diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ac7854f265..e4964e8909 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -23,7 +23,7 @@ "dbaeumer.vscode-eslint", "eamodio.gitlens", "editorconfig.editorconfig", - "vue.volar", + "vue.volar@2.2.10", "mrmlnc.vscode-duplicate", "ms-azuretools.vscode-docker", "ms-playwright.playwright", @@ -32,9 +32,8 @@ "mutantdino.resourcemonitor", "oderwat.indent-rainbow", "redhat.vscode-yaml", - "spmeesseman.vscode-taskexplorer", - "visualstudioexptteam.vscodeintellicode", - "ms-python.pylint" + "ms-python.pylint", + "charliermarsh.ruff" ], "settings": { "terminal.integrated.defaultProfile.linux": "zsh", diff --git a/.devcontainer/docker-compose.extend.yml b/.devcontainer/docker-compose.extend.yml index a92f42bc6d..ce1ce259fd 100644 --- a/.devcontainer/docker-compose.extend.yml +++ b/.devcontainer/docker-compose.extend.yml @@ -14,8 +14,8 @@ services: network_mode: service:db blobstore: ports: - - '9000' - - '9001' + - '9000:9000' + - '9001:9001' volumes: datatracker-vscode-ext: diff --git a/.github/ISSUE_TEMPLATE/new-feature.yml b/.github/ISSUE_TEMPLATE/new-feature.yml index ddf0b575e8..285081e1c8 100644 --- a/.github/ISSUE_TEMPLATE/new-feature.yml +++ b/.github/ISSUE_TEMPLATE/new-feature.yml @@ -1,6 +1,7 @@ name: Suggest new feature or enhancement description: Propose a new idea to be implemented. labels: ["enhancement"] +type: Feature body: - type: markdown attributes: diff --git a/.github/ISSUE_TEMPLATE/report-a-bug.yml b/.github/ISSUE_TEMPLATE/report-a-bug.yml index d5b725d721..47fa1185b4 100644 --- a/.github/ISSUE_TEMPLATE/report-a-bug.yml +++ b/.github/ISSUE_TEMPLATE/report-a-bug.yml @@ -1,6 +1,7 @@ name: Report a Datatracker bug description: Something in the datatracker's behavior isn't right? File a bug report. Don't use this to report RFC errata or issues with the content of Internet-Drafts. labels: ["bug"] +type: Bug body: - type: markdown attributes: diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml index 5f0a0d11b8..35172aa299 100644 --- a/.github/workflows/build-base-app.yml +++ b/.github/workflows/build-base-app.yml @@ -18,7 +18,7 @@ jobs: packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: token: ${{ secrets.GH_COMMON_TOKEN }} @@ -28,22 +28,22 @@ jobs: echo "IMGVERSION=$CURDATE" >> $GITHUB_ENV - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Docker Build & Push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 env: - DOCKER_BUILD_NO_SUMMARY: true + DOCKER_BUILD_SUMMARY: false with: context: . file: docker/base.Dockerfile @@ -51,6 +51,7 @@ jobs: push: true tags: | ghcr.io/ietf-tools/datatracker-app-base:${{ env.IMGVERSION }} + ghcr.io/ietf-tools/datatracker-app-base:py312 ${{ github.ref == 'refs/heads/main' && 'ghcr.io/ietf-tools/datatracker-app-base:latest' || '' }} - name: Update version references @@ -59,7 +60,7 @@ jobs: echo "${{ env.IMGVERSION }}" > dev/build/TARGET_BASE - name: Commit CHANGELOG.md - uses: stefanzweifel/git-auto-commit-action@v5 + uses: stefanzweifel/git-auto-commit-action@v7 with: branch: ${{ github.ref_name }} commit_message: 'ci: update base image target version to ${{ env.IMGVERSION }}' diff --git a/.github/workflows/build-celery-worker.yml b/.github/workflows/build-celery-worker.yml deleted file mode 100644 index d14e4f2c8b..0000000000 --- a/.github/workflows/build-celery-worker.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Build Celery Worker Docker Image - -on: - push: - branches: - - 'main' - paths: - - 'requirements.txt' - - 'dev/celery/**' - - '.github/workflows/build-celery-worker.yml' - - workflow_dispatch: - -jobs: - publish: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - - steps: - - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Docker Build & Push - uses: docker/build-push-action@v6 - env: - DOCKER_BUILD_NO_SUMMARY: true - with: - context: . - file: dev/celery/Dockerfile - platforms: linux/amd64,linux/arm64 - push: true - tags: ghcr.io/ietf-tools/datatracker-celery:latest - diff --git a/.github/workflows/build-devblobstore.yml b/.github/workflows/build-devblobstore.yml index 2d8214b448..14c4b1a135 100644 --- a/.github/workflows/build-devblobstore.yml +++ b/.github/workflows/build-devblobstore.yml @@ -20,22 +20,22 @@ jobs: packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Docker Build & Push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 env: - DOCKER_BUILD_NO_SUMMARY: true + DOCKER_BUILD_SUMMARY: false with: context: . file: docker/devblobstore.Dockerfile diff --git a/.github/workflows/build-mq-broker.yml b/.github/workflows/build-mq-broker.yml index 85c27c23cc..b297e34b47 100644 --- a/.github/workflows/build-mq-broker.yml +++ b/.github/workflows/build-mq-broker.yml @@ -24,31 +24,40 @@ jobs: packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Set rabbitmq version + id: rabbitmq-version + run: | + if [[ "${{ inputs.rabbitmq_version }}" == "" ]]; then + echo "RABBITMQ_VERSION=3.13-alpine" >> $GITHUB_OUTPUT + else + echo "RABBITMQ_VERSION=${{ inputs.rabbitmq_version }}" >> $GITHUB_OUTPUT + fi + - name: Docker Build & Push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 env: - DOCKER_BUILD_NO_SUMMARY: true + DOCKER_BUILD_SUMMARY: false with: context: . file: dev/mq/Dockerfile platforms: linux/amd64,linux/arm64 push: true - build-args: RABBITMQ_VERSION=${{ inputs.rabbitmq_version }} + build-args: RABBITMQ_VERSION=${{ steps.rabbitmq-version.outputs.RABBITMQ_VERSION }} tags: | - ghcr.io/ietf-tools/datatracker-mq:${{ inputs.rabbitmq_version }} + ghcr.io/ietf-tools/datatracker-mq:${{ steps.rabbitmq-version.outputs.RABBITMQ_VERSION }} ghcr.io/ietf-tools/datatracker-mq:latest diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 36439aa6cf..a89bac46e7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,13 +16,13 @@ on: - Skip - Staging Only - Staging + Prod - sandbox: - description: 'Deploy to Sandbox' + dev: + description: 'Deploy to Dev' default: true required: true type: boolean - sandboxNoDbRefresh: - description: 'Sandbox Disable Daily DB Refresh' + devNoDbRefresh: + description: 'Dev Disable Daily DB Refresh' default: false required: true type: boolean @@ -66,7 +66,7 @@ jobs: base_image_version: ${{ steps.baseimgversion.outputs.base_image_version }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 1 fetch-tags: false @@ -79,6 +79,7 @@ jobs: token: ${{ github.token }} branch: release skipInvalidTags: true + patchList: fix, bugfix, perf, refactor, test, tests, chore - name: Get Dev Version if: ${{ github.ref_name != 'release' }} @@ -97,7 +98,7 @@ jobs: echo "IS_RELEASE=true" >> $GITHUB_ENV - name: Create Draft Release - uses: ncipollo/release-action@v1.16.0 + uses: ncipollo/release-action@v1.21.0 if: ${{ github.ref_name == 'release' }} with: prerelease: true @@ -163,18 +164,18 @@ jobs: TARGET_BASE: ${{needs.prepare.outputs.base_image_version}} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 1 fetch-tags: false - name: Setup Node.js environment - uses: actions/setup-node@v4 + uses: actions/setup-node@v6 with: node-version: 18.x - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.x" @@ -185,7 +186,7 @@ jobs: - name: Download a Coverage Results if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - uses: actions/download-artifact@v4.1.9 + uses: actions/download-artifact@v8.0.1 with: name: coverage @@ -252,10 +253,10 @@ jobs: EOL - name: Setup Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} @@ -266,7 +267,7 @@ jobs: run: echo "FEATURE_LATEST_TAG=$(echo $GITHUB_REF_NAME | tr / -)" >> $GITHUB_ENV - name: Build Images - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 env: DOCKER_BUILD_SUMMARY: false with: @@ -277,8 +278,6 @@ jobs: tags: | ghcr.io/ietf-tools/datatracker:${{ env.PKG_VERSION }} ${{ env.FEATURE_LATEST_TAG && format('ghcr.io/ietf-tools/datatracker:{0}-latest', env.FEATURE_LATEST_TAG) || null }} - cache-from: type=gha - cache-to: type=gha,mode=max - name: Update CHANGELOG id: changelog @@ -292,7 +291,7 @@ jobs: - name: Download Coverage Results if: ${{ github.event.inputs.skiptests == 'false' || github.ref_name == 'release' }} - uses: actions/download-artifact@v4.1.9 + uses: actions/download-artifact@v8.0.1 with: name: coverage @@ -316,7 +315,7 @@ jobs: histCoveragePath: historical-coverage.json - name: Create Release - uses: ncipollo/release-action@v1.16.0 + uses: ncipollo/release-action@v1.21.0 if: ${{ env.SHOULD_DEPLOY == 'true' }} with: allowUpdates: true @@ -329,7 +328,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} - name: Update Baseline Coverage - uses: ncipollo/release-action@v1.16.0 + uses: ncipollo/release-action@v1.21.0 if: ${{ github.event.inputs.updateCoverage == 'true' || github.ref_name == 'release' }} with: allowUpdates: true @@ -342,7 +341,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} - name: Upload Build Artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: release-${{ env.PKG_VERSION }} path: /home/runner/work/release/release.tar.gz @@ -361,7 +360,7 @@ jobs: steps: - name: Notify on Slack (Success) if: ${{ !contains(join(needs.*.result, ','), 'failure') }} - uses: slackapi/slack-github-action@v2 + uses: slackapi/slack-github-action@v3 with: token: ${{ secrets.SLACK_GH_BOT }} method: chat.postMessage @@ -376,7 +375,7 @@ jobs: value: "Completed" - name: Notify on Slack (Failure) if: ${{ contains(join(needs.*.result, ','), 'failure') }} - uses: slackapi/slack-github-action@v2 + uses: slackapi/slack-github-action@v3 with: token: ${{ secrets.SLACK_GH_BOT }} method: chat.postMessage @@ -391,44 +390,42 @@ jobs: value: "Failed" # ----------------------------------------------------------------- - # SANDBOX + # DEV # ----------------------------------------------------------------- - sandbox: - name: Deploy to Sandbox - if: ${{ !failure() && !cancelled() && github.event.inputs.sandbox == 'true' }} + dev: + name: Deploy to Dev + if: ${{ !failure() && !cancelled() && github.event.inputs.dev == 'true' }} needs: [prepare, release] - runs-on: [self-hosted, dev-server] + runs-on: ubuntu-latest environment: - name: sandbox + name: dev env: PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} steps: - - uses: actions/checkout@v4 - - - name: Download a Release Artifact - uses: actions/download-artifact@v4.1.9 - with: - name: release-${{ env.PKG_VERSION }} - - - name: Deploy to containers - env: - DEBIAN_FRONTEND: noninteractive - run: | - echo "Reset production flags in settings.py..." - sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = True/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'development'/" ietf/settings.py - echo "Install Deploy to Container CLI dependencies..." - cd dev/deploy-to-container - npm ci - cd ../.. - echo "Start Deploy..." - node ./dev/deploy-to-container/cli.js --branch ${{ github.ref_name }} --domain dev.ietf.org --appversion ${{ env.PKG_VERSION }} --commit ${{ github.sha }} --ghrunid ${{ github.run_id }} --nodbrefresh ${{ github.event.inputs.sandboxNoDbRefresh }} - - - name: Cleanup old docker resources - env: - DEBIAN_FRONTEND: noninteractive - run: | - docker image prune -a -f + - uses: actions/checkout@v6 + with: + ref: main + + - name: Get Deploy Name + env: + DEBIAN_FRONTEND: noninteractive + run: | + echo "Install Get Deploy Name CLI dependencies..." + cd dev/k8s-get-deploy-name + npm ci + echo "Get Deploy Name..." + echo "DEPLOY_NAMESPACE=$(node cli.js --branch ${{ github.ref_name }})" >> "$GITHUB_ENV" + + - name: Deploy to dev + uses: ietf-tools/workflow-dispatch-action@v1 + with: + workflow: deploy-dev.yml + repo: ietf-tools/infra-k8s + ref: main + token: ${{ secrets.GH_INFRA_K8S_TOKEN }} + inputs: '{ "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}", "namespace":"${{ env.DEPLOY_NAMESPACE }}", "disableDailyDbRefresh":${{ inputs.devNoDbRefresh }} }' + waitForCompletionTimeout: 60m # ----------------------------------------------------------------- # STAGING @@ -445,30 +442,24 @@ jobs: steps: - name: Refresh Staging DB - uses: the-actions-org/workflow-dispatch@v4 + uses: ietf-tools/workflow-dispatch-action@v1 with: - workflow: update-staging-db.yml + workflow: deploy-db.yml repo: ietf-tools/infra-k8s ref: main token: ${{ secrets.GH_INFRA_K8S_TOKEN }} - inputs: '{ "sourceDb":"datatracker" }' - wait-for-completion: true - wait-for-completion-timeout: 5m - wait-for-completion-interval: 20s - display-workflow-run-url: false + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "manifest":"postgres", "forceRecreate":true, "restoreToLastFullSnapshot":true, "waitClusterReady":true }' + waitForCompletionTimeout: 120m - name: Deploy to staging - uses: the-actions-org/workflow-dispatch@v4 + uses: ietf-tools/workflow-dispatch-action@v1 with: workflow: deploy.yml repo: ietf-tools/infra-k8s ref: main token: ${{ secrets.GH_INFRA_K8S_TOKEN }} inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}" }' - wait-for-completion: true - wait-for-completion-timeout: 10m - wait-for-completion-interval: 30s - display-workflow-run-url: false + waitForCompletionTimeout: 30m # ----------------------------------------------------------------- # PROD @@ -485,14 +476,11 @@ jobs: steps: - name: Deploy to production - uses: the-actions-org/workflow-dispatch@v4 + uses: ietf-tools/workflow-dispatch-action@v1 with: workflow: deploy.yml repo: ietf-tools/infra-k8s ref: main token: ${{ secrets.GH_INFRA_K8S_TOKEN }} inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}" }' - wait-for-completion: true - wait-for-completion-timeout: 10m - wait-for-completion-interval: 30s - display-workflow-run-url: false + waitForCompletionTimeout: 30m diff --git a/.github/workflows/ci-run-tests.yml b/.github/workflows/ci-run-tests.yml index 278bd8af2f..5349f1ac7a 100644 --- a/.github/workflows/ci-run-tests.yml +++ b/.github/workflows/ci-run-tests.yml @@ -23,7 +23,7 @@ jobs: base_image_version: ${{ steps.baseimgversion.outputs.base_image_version }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: fetch-depth: 1 fetch-tags: false diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 3444c03b5e..bc20779ae6 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -26,12 +26,12 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Initialize CodeQL - uses: github/codeql-action/init@v3 + uses: github/codeql-action/init@v4 with: languages: ${{ matrix.language }} - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v3 + uses: github/codeql-action/analyze@v4 diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 6d0683c471..e255b270ff 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: 'Checkout Repository' - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: 'Dependency Review' uses: actions/dependency-review-action@v4 with: diff --git a/.github/workflows/dev-assets-sync-nightly.yml b/.github/workflows/dev-assets-sync-nightly.yml index a7fe67f012..cd986f06f3 100644 --- a/.github/workflows/dev-assets-sync-nightly.yml +++ b/.github/workflows/dev-assets-sync-nightly.yml @@ -29,35 +29,21 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Login to GitHub Container Registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Docker Build & Push - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 env: - DOCKER_BUILD_NO_SUMMARY: true + DOCKER_BUILD_SUMMARY: false with: context: . file: dev/shared-assets-sync/Dockerfile push: true tags: ghcr.io/ietf-tools/datatracker-rsync-assets:latest - - sync: - name: Run assets rsync - if: ${{ always() }} - runs-on: [self-hosted, dev-server] - needs: [build] - steps: - - name: Run rsync - env: - DEBIAN_FRONTEND: noninteractive - run: | - docker pull ghcr.io/ietf-tools/datatracker-rsync-assets:latest - docker run --rm -v dt-assets:/assets ghcr.io/ietf-tools/datatracker-rsync-assets:latest - docker image prune -a -f diff --git a/.github/workflows/sandbox-refresh.yml b/.github/workflows/sandbox-refresh.yml deleted file mode 100644 index 3ddb119e4f..0000000000 --- a/.github/workflows/sandbox-refresh.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Sandbox Refresh - -on: - # Run every night - schedule: - - cron: '0 9 * * *' - - workflow_dispatch: - -jobs: - main: - name: Refresh DBs - runs-on: [self-hosted, dev-server] - permissions: - contents: read - - steps: - - uses: actions/checkout@v4 - - - name: Refresh DBs - env: - DEBIAN_FRONTEND: noninteractive - run: | - echo "Install Deploy to Container CLI dependencies..." - cd dev/deploy-to-container - npm ci - cd ../.. - echo "Start Refresh..." - node ./dev/deploy-to-container/refresh.js - - - name: Cleanup old docker resources - env: - DEBIAN_FRONTEND: noninteractive - run: | - docker image prune -a -f diff --git a/.github/workflows/tests-az.yml b/.github/workflows/tests-az.yml index 8553563a19..833ca89bef 100644 --- a/.github/workflows/tests-az.yml +++ b/.github/workflows/tests-az.yml @@ -38,7 +38,7 @@ jobs: ssh-keyscan -t rsa $vminfo >> ~/.ssh/known_hosts - name: Remote SSH into VM - uses: appleboy/ssh-action@2ead5e36573f08b82fbfce1504f1a4b05a647c6f + uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f10c1db9a3..ad2e35408d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,7 +32,7 @@ jobs: image: ghcr.io/ietf-tools/datatracker-devblobstore:latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Prepare for tests run: | @@ -68,14 +68,14 @@ jobs: coverage xml - name: Upload geckodriver.log - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 if: ${{ failure() }} with: name: geckodriverlog path: geckodriver.log - name: Upload Coverage Results to Codecov - uses: codecov/codecov-action@v5 + uses: codecov/codecov-action@v6 with: disable_search: true files: coverage.xml @@ -87,7 +87,7 @@ jobs: mv latest-coverage.json coverage.json - name: Upload Coverage Results as Build Artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 if: ${{ always() }} with: name: coverage @@ -102,9 +102,9 @@ jobs: project: [chromium, firefox] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v6 with: node-version: '18' @@ -121,7 +121,7 @@ jobs: npx playwright test --project=${{ matrix.project }} - name: Upload Report - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 if: ${{ always() }} continue-on-error: true with: @@ -130,6 +130,7 @@ jobs: if-no-files-found: ignore tests-playwright-legacy: + if: ${{ false }} # disable until we sort out suspected test runner issue name: Playwright Legacy Tests runs-on: ubuntu-latest container: ghcr.io/ietf-tools/datatracker-app-base:${{ inputs.targetBaseVersion }} @@ -143,7 +144,7 @@ jobs: image: ghcr.io/ietf-tools/datatracker-db:latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 - name: Prepare for tests run: | @@ -180,7 +181,7 @@ jobs: npx playwright test --project=${{ matrix.project }} -c playwright-legacy.config.js - name: Upload Report - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 if: ${{ always() }} continue-on-error: true with: diff --git a/.gitignore b/.gitignore index 84bc800e3b..ccc7a46b08 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .DS_store datatracker.sublime-project datatracker.sublime-workspace +/.claude /.coverage /.factoryboy_random_state /.mypy_cache diff --git a/.pnp.cjs b/.pnp.cjs index 5fcce34d2f..6c76263c7e 100644 --- a/.pnp.cjs +++ b/.pnp.cjs @@ -42,6 +42,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["@fullcalendar/luxon3", "virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:6.1.11"],\ ["@fullcalendar/timegrid", "virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:6.1.11"],\ ["@fullcalendar/vue3", "virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:6.1.11"],\ + ["@kurkle/color", "npm:0.3.1"],\ ["@parcel/optimizer-data-url", "npm:2.12.0"],\ ["@parcel/transformer-inline-string", "npm:2.12.0"],\ ["@parcel/transformer-sass", "npm:2.12.0"],\ @@ -56,6 +57,9 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["browserlist", "npm:1.0.1"],\ ["c8", "npm:9.1.0"],\ ["caniuse-lite", "npm:1.0.30001603"],\ + ["chart.js", "npm:4.5.1"],\ + ["chartjs-plugin-autocolors", "virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:0.3.1"],\ + ["chartjs-plugin-zoom", "virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:2.2.0"],\ ["d3", "npm:7.9.0"],\ ["eslint", "npm:8.57.0"],\ ["eslint-config-standard", "virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:17.1.0"],\ @@ -883,6 +887,22 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { "linkType": "HARD"\ }]\ ]],\ + ["@kurkle/color", [\ + ["npm:0.3.1", {\ + "packageLocation": "./.yarn/cache/@kurkle-color-npm-0.3.1-174f3d038c-e6be5c081b.zip/node_modules/@kurkle/color/",\ + "packageDependencies": [\ + ["@kurkle/color", "npm:0.3.1"]\ + ],\ + "linkType": "HARD"\ + }],\ + ["npm:0.3.4", {\ + "packageLocation": "./.yarn/cache/@kurkle-color-npm-0.3.4-fbd637031f-b95c6abe02.zip/node_modules/@kurkle/color/",\ + "packageDependencies": [\ + ["@kurkle/color", "npm:0.3.4"]\ + ],\ + "linkType": "HARD"\ + }]\ + ]],\ ["@lezer/common", [\ ["npm:0.15.12", {\ "packageLocation": "./.yarn/cache/@lezer-common-npm-0.15.12-62017272b0-dae6581618.zip/node_modules/@lezer/common/",\ @@ -2616,6 +2636,15 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { "linkType": "HARD"\ }]\ ]],\ + ["@types/hammerjs", [\ + ["npm:2.0.46", {\ + "packageLocation": "./.yarn/cache/@types-hammerjs-npm-2.0.46-de99d4d9d1-caba6ec788.zip/node_modules/@types/hammerjs/",\ + "packageDependencies": [\ + ["@types/hammerjs", "npm:2.0.46"]\ + ],\ + "linkType": "HARD"\ + }]\ + ]],\ ["@types/istanbul-lib-coverage", [\ ["npm:2.0.4", {\ "packageLocation": "./.yarn/cache/@types-istanbul-lib-coverage-npm-2.0.4-734954bb56-a25d7589ee.zip/node_modules/@types/istanbul-lib-coverage/",\ @@ -3545,6 +3574,66 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { "linkType": "HARD"\ }]\ ]],\ + ["chart.js", [\ + ["npm:4.5.1", {\ + "packageLocation": "./.yarn/cache/chart.js-npm-4.5.1-97698d58cc-34b35b3736.zip/node_modules/chart.js/",\ + "packageDependencies": [\ + ["chart.js", "npm:4.5.1"],\ + ["@kurkle/color", "npm:0.3.4"]\ + ],\ + "linkType": "HARD"\ + }]\ + ]],\ + ["chartjs-plugin-autocolors", [\ + ["npm:0.3.1", {\ + "packageLocation": "./.yarn/cache/chartjs-plugin-autocolors-npm-0.3.1-7e93d38139-de4f87b5bb.zip/node_modules/chartjs-plugin-autocolors/",\ + "packageDependencies": [\ + ["chartjs-plugin-autocolors", "npm:0.3.1"]\ + ],\ + "linkType": "SOFT"\ + }],\ + ["virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:0.3.1", {\ + "packageLocation": "./.yarn/__virtual__/chartjs-plugin-autocolors-virtual-6e228c1a1e/0/cache/chartjs-plugin-autocolors-npm-0.3.1-7e93d38139-de4f87b5bb.zip/node_modules/chartjs-plugin-autocolors/",\ + "packageDependencies": [\ + ["chartjs-plugin-autocolors", "virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:0.3.1"],\ + ["@kurkle/color", "npm:0.3.1"],\ + ["@types/chart.js", null],\ + ["@types/kurkle__color", null],\ + ["chart.js", "npm:4.5.1"]\ + ],\ + "packagePeers": [\ + "@kurkle/color",\ + "@types/chart.js",\ + "@types/kurkle__color",\ + "chart.js"\ + ],\ + "linkType": "HARD"\ + }]\ + ]],\ + ["chartjs-plugin-zoom", [\ + ["npm:2.2.0", {\ + "packageLocation": "./.yarn/cache/chartjs-plugin-zoom-npm-2.2.0-85aea0b81e-a540e38340.zip/node_modules/chartjs-plugin-zoom/",\ + "packageDependencies": [\ + ["chartjs-plugin-zoom", "npm:2.2.0"]\ + ],\ + "linkType": "SOFT"\ + }],\ + ["virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:2.2.0", {\ + "packageLocation": "./.yarn/__virtual__/chartjs-plugin-zoom-virtual-45332d2c47/0/cache/chartjs-plugin-zoom-npm-2.2.0-85aea0b81e-a540e38340.zip/node_modules/chartjs-plugin-zoom/",\ + "packageDependencies": [\ + ["chartjs-plugin-zoom", "virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:2.2.0"],\ + ["@types/chart.js", null],\ + ["@types/hammerjs", "npm:2.0.46"],\ + ["chart.js", "npm:4.5.1"],\ + ["hammerjs", "npm:2.0.8"]\ + ],\ + "packagePeers": [\ + "@types/chart.js",\ + "chart.js"\ + ],\ + "linkType": "HARD"\ + }]\ + ]],\ ["chokidar", [\ ["npm:3.5.3", {\ "packageLocation": "./.yarn/cache/chokidar-npm-3.5.3-c5f9b0a56a-b49fcde401.zip/node_modules/chokidar/",\ @@ -5709,6 +5798,15 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { "linkType": "HARD"\ }]\ ]],\ + ["hammerjs", [\ + ["npm:2.0.8", {\ + "packageLocation": "./.yarn/cache/hammerjs-npm-2.0.8-f656ba2573-b092da7d15.zip/node_modules/hammerjs/",\ + "packageDependencies": [\ + ["hammerjs", "npm:2.0.8"]\ + ],\ + "linkType": "HARD"\ + }]\ + ]],\ ["has", [\ ["npm:1.0.3", {\ "packageLocation": "./.yarn/cache/has-npm-1.0.3-b7f00631c1-b9ad53d53b.zip/node_modules/has/",\ @@ -8326,6 +8424,7 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["@fullcalendar/luxon3", "virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:6.1.11"],\ ["@fullcalendar/timegrid", "virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:6.1.11"],\ ["@fullcalendar/vue3", "virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:6.1.11"],\ + ["@kurkle/color", "npm:0.3.1"],\ ["@parcel/optimizer-data-url", "npm:2.12.0"],\ ["@parcel/transformer-inline-string", "npm:2.12.0"],\ ["@parcel/transformer-sass", "npm:2.12.0"],\ @@ -8340,6 +8439,9 @@ function $$SETUP_STATE(hydrateRuntimeState, basePath) { ["browserlist", "npm:1.0.1"],\ ["c8", "npm:9.1.0"],\ ["caniuse-lite", "npm:1.0.30001603"],\ + ["chart.js", "npm:4.5.1"],\ + ["chartjs-plugin-autocolors", "virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:0.3.1"],\ + ["chartjs-plugin-zoom", "virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:2.2.0"],\ ["d3", "npm:7.9.0"],\ ["eslint", "npm:8.57.0"],\ ["eslint-config-standard", "virtual:dc3fc578bfa5e06182a4d2be39ede0bc5b74940b1ffe0d70c26892ab140a4699787750fba175dc306292e80b4aa2c8c5f68c2a821e69b2c37e360c0dff36ff66#npm:17.1.0"],\ diff --git a/.vscode/settings.json b/.vscode/settings.json index b0ceba5c9d..ad6b0adc84 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -56,5 +56,9 @@ "python.linting.pylintArgs": ["--load-plugins", "pylint_django"], "python.testing.pytestEnabled": false, "python.testing.unittestEnabled": false, - "python.linting.enabled": true + "python.linting.enabled": true, + "python.terminal.shellIntegration.enabled": false, + "vs-kubernetes": { + "disable-linters": ["resource-limits"] + } } diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 4bd0b99363..8b36b0e6ac 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -105,10 +105,11 @@ "command": "/usr/local/bin/python", "args": [ "-m", - "smtpd", + "aiosmtpd", "-n", "-c", - "DebuggingServer", + "ietf.utils.aiosmtpd.DevDebuggingHandler", + "-l", "localhost:2025" ], "presentation": { diff --git a/README.md b/README.md index abebb7ca02..baffc311e7 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,10 @@ [![Release](https://img.shields.io/github/release/ietf-tools/datatracker.svg?style=flat&maxAge=300)](https://github.com/ietf-tools/datatracker/releases) [![License](https://img.shields.io/github/license/ietf-tools/datatracker)](https://github.com/ietf-tools/datatracker/blob/main/LICENSE) [![Code Coverage](https://codecov.io/gh/ietf-tools/datatracker/branch/feat/bs5/graph/badge.svg?token=V4DXB0Q28C)](https://codecov.io/gh/ietf-tools/datatracker) -[![Python Version](https://img.shields.io/badge/python-3.9-blue?logo=python&logoColor=white)](#prerequisites) +[![Python Version](https://img.shields.io/badge/python-3.12-blue?logo=python&logoColor=white)](#prerequisites) [![Django Version](https://img.shields.io/badge/django-4.x-51be95?logo=django&logoColor=white)](#prerequisites) [![Node Version](https://img.shields.io/badge/node.js-16.x-green?logo=node.js&logoColor=white)](#prerequisites) -[![MariaDB Version](https://img.shields.io/badge/postgres-16-blue?logo=postgresql&logoColor=white)](#prerequisites) +[![MariaDB Version](https://img.shields.io/badge/postgres-17-blue?logo=postgresql&logoColor=white)](#prerequisites) ##### The day-to-day front-end to the IETF database for people who work on IETF standards. @@ -142,7 +142,7 @@ Pages will gradually be updated to Vue 3 components. These components are locate Each Vue 3 app has its own sub-directory. For example, the agenda app is located under `/client/agenda`. -The datatracker makes use of the Django-Vite plugin to point to either the Vite.js server or the precompiled production files. The `DJANGO_VITE_DEV_MODE` flag, found in the `ietf/settings_local.py` file determines whether the Vite.js server is used or not. +The datatracker makes use of the Django-Vite plugin to point to either the Vite.js server or the precompiled production files. The `DJANGO_VITE["default"]["dev_mode"]` flag, found in the `ietf/settings_local.py` file determines whether the Vite.js server is used or not. In development mode, you must start the Vite.js development server, in addition to the usual Datatracker server: diff --git a/client/agenda/AgendaDetailsModal.vue b/client/agenda/AgendaDetailsModal.vue index 2582bf2159..69c8ef8b53 100644 --- a/client/agenda/AgendaDetailsModal.vue +++ b/client/agenda/AgendaDetailsModal.vue @@ -274,6 +274,7 @@ async function fetchSessionMaterials () { diff --git a/client/agenda/AgendaScheduleList.vue b/client/agenda/AgendaScheduleList.vue index d38c7842eb..bbe5dfee8b 100644 --- a/client/agenda/AgendaScheduleList.vue +++ b/client/agenda/AgendaScheduleList.vue @@ -302,7 +302,7 @@ const meetingEvents = computed(() => { icon: 'collection', href: undefined, click: () => showMaterials(item.id), - color: 'black' + color: 'darkgray' }) links.push({ id: `lnk-${item.id}-tar`, @@ -398,16 +398,6 @@ const meetingEvents = computed(() => { color: 'teal' }) } - // -> Calendar item - if (item.links.calendar) { - links.push({ - id: `lnk-${item.id}-calendar`, - label: 'Calendar (.ics) entry for this session', - icon: 'calendar-check', - href: item.links.calendar, - color: 'pink' - }) - } } else { // -> Post event if (meetingNumberInt >= 60) { @@ -484,6 +474,16 @@ const meetingEvents = computed(() => { } } } + // Add Calendar item for all events that has a calendar link + if (item.adjustedEnd > current && item.links.calendar) { + links.push({ + id: `lnk-${item.id}-calendar`, + label: 'Calendar (.ics) entry for this session', + icon: 'calendar-check', + href: item.links.calendar, + color: 'pink' + }) + } // Event icon let icon = null @@ -1155,7 +1155,7 @@ onBeforeUnmount(() => { .agenda-table-cell-links-buttons { white-space: nowrap; - > a, > i { + > a, > i, > button { margin-left: 3px; color: #666; cursor: pointer; @@ -1197,6 +1197,18 @@ onBeforeUnmount(() => { background-color: rgba($orange-500, .3); } } + &.text-darkgray { + color: $gray-900; + background-color: rgba($gray-700, .1); + + @at-root .theme-dark & { + color: $gray-100; + } + + &:hover, &:focus { + background-color: rgba($gray-700, .3); + } + } &.text-blue { color: $blue-600; background-color: rgba($blue-300, .1); diff --git a/client/agenda/store.js b/client/agenda/store.js index 359c5fbf05..b5498303a6 100644 --- a/client/agenda/store.js +++ b/client/agenda/store.js @@ -141,7 +141,7 @@ export const useAgendaStore = defineStore('agenda', { meetingNumber = meetingData.meetingNumber } - const resp = await fetch(`/api/meeting/${meetingNumber}/agenda-data`, { credentials: 'omit' }) + const resp = await fetch(`/api/meeting/${meetingNumber}/agenda-data`) if (!resp.ok) { throw new Error(resp.statusText) } diff --git a/client/components/ChatLog.vue b/client/components/ChatLog.vue index f9dc382bfe..b3a4f7b40f 100644 --- a/client/components/ChatLog.vue +++ b/client/components/ChatLog.vue @@ -159,4 +159,18 @@ onMounted(() => { } } } + +[data-bs-theme="dark"] .chatlog { + .n-timeline-item-content__title { + color: #d63384 !important; + } + + .n-timeline-item-content__content { + color: #fff !important; + } + + .n-timeline-item-content__meta { + color: #0569ffd9 !important; + } +} diff --git a/client/components/Polls.vue b/client/components/Polls.vue index 30cc9e8f36..0846d4ed16 100644 --- a/client/components/Polls.vue +++ b/client/components/Polls.vue @@ -90,3 +90,21 @@ onMounted(() => { }) + diff --git a/client/shared/xslugify.js b/client/shared/xslugify.js index e1f7a34483..e1ac556ddf 100644 --- a/client/shared/xslugify.js +++ b/client/shared/xslugify.js @@ -1,5 +1,5 @@ import slugify from 'slugify' export default (str) => { - return slugify(str.replaceAll('/', '-'), { lower: true }) + return slugify(str.replaceAll('/', '-').replaceAll(/['&]/g, ''), { lower: true }) } diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index cd0a70667c..1e44f47761 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20250402T1611 +FROM ghcr.io/ietf-tools/datatracker-app-base:20260527T1529 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index 1195fc9a0b..0284151019 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20250402T1611 +20260527T1529 diff --git a/dev/build/celery-start.sh b/dev/build/celery-start.sh index f6284e84db..69dcd7bbda 100644 --- a/dev/build/celery-start.sh +++ b/dev/build/celery-start.sh @@ -5,10 +5,29 @@ echo "Running Datatracker checks..." ./ietf/manage.py check -if ! ietf/manage.py migrate --check ; then +# Check whether the blobdb database exists - inspectdb will return a false +# status if not. +if ietf/manage.py inspectdb --database blobdb > /dev/null 2>&1; then + HAVE_BLOBDB="yes" +fi + +migrations_applied_for () { + local DATABASE=${1:-default} + ietf/manage.py migrate --check --database "$DATABASE" +} + +migrations_all_applied () { + if [[ "$HAVE_BLOBDB" == "yes" ]]; then + migrations_applied_for default && migrations_applied_for blobdb + else + migrations_applied_for default + fi +} + +if ! migrations_all_applied; then echo "Unapplied migrations found, waiting to start..." sleep 5 - while ! ietf/manage.py migrate --check ; do + while ! migrations_all_applied ; do echo "... still waiting for migrations..." sleep 5 done diff --git a/dev/build/datatracker-start.sh b/dev/build/datatracker-start.sh index 5eb99c4745..012a563412 100644 --- a/dev/build/datatracker-start.sh +++ b/dev/build/datatracker-start.sh @@ -3,10 +3,29 @@ echo "Running Datatracker checks..." ./ietf/manage.py check -if ! ietf/manage.py migrate --check ; then +# Check whether the blobdb database exists - inspectdb will return a false +# status if not. +if ietf/manage.py inspectdb --database blobdb > /dev/null 2>&1; then + HAVE_BLOBDB="yes" +fi + +migrations_applied_for () { + local DATABASE=${1:-default} + ietf/manage.py migrate --check --database "$DATABASE" +} + +migrations_all_applied () { + if [[ "$HAVE_BLOBDB" == "yes" ]]; then + migrations_applied_for default && migrations_applied_for blobdb + else + migrations_applied_for default + fi +} + +if ! migrations_all_applied; then echo "Unapplied migrations found, waiting to start..." sleep 5 - while ! ietf/manage.py migrate --check ; do + while ! migrations_all_applied ; do echo "... still waiting for migrations..." sleep 5 done @@ -26,16 +45,6 @@ cleanup () { trap 'trap "" TERM; cleanup' TERM # start gunicorn in the background so we can trap the TERM signal -gunicorn \ - -c /workspace/gunicorn.conf.py \ - --workers "${DATATRACKER_GUNICORN_WORKERS:-9}" \ - --max-requests "${DATATRACKER_GUNICORN_MAX_REQUESTS:-32768}" \ - --timeout "${DATATRACKER_GUNICORN_TIMEOUT:-180}" \ - --bind :8000 \ - --log-level "${DATATRACKER_GUNICORN_LOG_LEVEL:-info}" \ - --capture-output \ - --access-logfile -\ - ${DATATRACKER_GUNICORN_EXTRA_ARGS} \ - ietf.wsgi:application & +gunicorn -c /workspace/gunicorn.conf.py ${DATATRACKER_GUNICORN_EXTRA_ARGS} ietf.wsgi:application & gunicorn_pid=$! wait "${gunicorn_pid}" diff --git a/dev/build/gunicorn.conf.py b/dev/build/gunicorn.conf.py index 032d95ee0d..03e81eac5e 100644 --- a/dev/build/gunicorn.conf.py +++ b/dev/build/gunicorn.conf.py @@ -1,4 +1,33 @@ -# Copyright The IETF Trust 2024, All Rights Reserved +# Copyright The IETF Trust 2024-2026, All Rights Reserved + +import os +import ietf +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.instrumentation.django import DjangoInstrumentor +from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor +from opentelemetry.instrumentation.pymemcache import PymemcacheInstrumentor +from opentelemetry.instrumentation.requests import RequestsInstrumentor + +# Bind all ipv4 interfaces (nginx uses loopback, but k8s health checks don't) +_BIND_PORT = os.environ.get("DATATRACKER_GUNICORN_BIND_PORT", "8000") +bind = [f"0.0.0.0:{_BIND_PORT}"] + +# Disable control socket +control_socket_disable = True + +# Settings configurable via environment +workers = int(os.environ.get("DATATRACKER_GUNICORN_WORKERS", "9")) +max_requests = int(os.environ.get("DATATRACKER_GUNICORN_MAX_REQUESTS", "32768")) +timeout = int(os.environ.get("DATATRACKER_GUNICORN_TIMEOUT", "180")) +loglevel = os.environ.get("DATATRACKER_GUNICORN_LOG_LEVEL", "info") + +# Logging / stdout capture +capture_output = True +accesslog = "-" # Configure security scheme headers for forwarded requests. Cloudflare sets X-Forwarded-Proto # for us. Don't trust any of the other similar headers. Only trust the header if it's coming @@ -119,3 +148,34 @@ def post_request(worker, req, environ, resp): in_flight = in_flight_by_pid.get(worker.pid, []) if request_description in in_flight: in_flight.remove(request_description) + +def post_fork(server, worker): + server.log.info("Worker spawned (pid: %s)", worker.pid) + + # Setting DATATRACKER_OPENTELEMETRY_ENABLE=all in the environment will enable all + # opentelemetry instrumentations. Individual instrumentations can be selected by + # using a space-separated list. See the code below for available instrumentations. + telemetry_env = os.environ.get("DATATRACKER_OPENTELEMETRY_ENABLE", "").strip() + if telemetry_env != "": + enabled_telemetry = [tok.strip().lower() for tok in telemetry_env.split()] + resource = Resource.create(attributes={ + "service.name": "datatracker", + "service.version": ietf.__version__, + "service.instance.id": worker.pid, + "service.namespace": "datatracker", + "deployment.environment.name": os.environ.get("DATATRACKER_SERVICE_ENV", "dev") + }) + trace.set_tracer_provider(TracerProvider(resource=resource)) + otlp_exporter = OTLPSpanExporter(endpoint="https://heimdall-otlp.ietf.org/v1/traces") + + trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(otlp_exporter)) + + # Instrumentations + if "all" in enabled_telemetry or "django" in enabled_telemetry: + DjangoInstrumentor().instrument() + if "all" in enabled_telemetry or "psycopg2" in enabled_telemetry: + Psycopg2Instrumentor().instrument() + if "all" in enabled_telemetry or "pymemcache" in enabled_telemetry: + PymemcacheInstrumentor().instrument() + if "all" in enabled_telemetry or "requests" in enabled_telemetry: + RequestsInstrumentor().instrument() diff --git a/dev/build/migration-start.sh b/dev/build/migration-start.sh index 211638aec6..578daf5cef 100644 --- a/dev/build/migration-start.sh +++ b/dev/build/migration-start.sh @@ -3,4 +3,11 @@ echo "Running Datatracker migrations..." ./ietf/manage.py migrate --settings=settings_local +# Check whether the blobdb database exists - inspectdb will return a false +# status if not. +if ./ietf/manage.py inspectdb --database blobdb > /dev/null 2>&1; then + echo "Running Blobdb migrations ..." + ./ietf/manage.py migrate --settings=settings_local --database=blobdb +fi + echo "Done!" diff --git a/dev/build/start.sh b/dev/build/start.sh index ef6c7fc0d9..3b03637068 100644 --- a/dev/build/start.sh +++ b/dev/build/start.sh @@ -2,7 +2,7 @@ # # Environment config: # -# CONTAINER_ROLE - datatracker, celery, or beat (defaults to datatracker) +# CONTAINER_ROLE - datatracker, celery, beat, migrations, or replicator (defaults to datatracker) # case "${CONTAINER_ROLE:-datatracker}" in auth) @@ -20,6 +20,9 @@ case "${CONTAINER_ROLE:-datatracker}" in migrations) exec ./migration-start.sh ;; + replicator) + exec ./celery-start.sh --app=ietf worker --queues=blobdb --concurrency=1 + ;; *) echo "Unknown role '${CONTAINER_ROLE}'" exit 255 diff --git a/dev/celery/Dockerfile b/dev/celery/Dockerfile index 12eb15eb81..e69de29bb2 100644 --- a/dev/celery/Dockerfile +++ b/dev/celery/Dockerfile @@ -1,23 +0,0 @@ -# Dockerfile for celery worker -# -FROM ghcr.io/ietf-tools/datatracker-app-base:latest -LABEL maintainer="IETF Tools Team " - -ENV DEBIAN_FRONTEND=noninteractive - -RUN apt-get purge -y imagemagick imagemagick-6-common - -# Copy the startup file -COPY dev/celery/docker-init.sh /docker-init.sh -RUN sed -i 's/\r$//' /docker-init.sh && \ - chmod +x /docker-init.sh - -# Install current datatracker python dependencies -COPY requirements.txt /tmp/pip-tmp/ -RUN pip3 --disable-pip-version-check --no-cache-dir install -r /tmp/pip-tmp/requirements.txt -RUN rm -rf /tmp/pip-tmp - -# Add watchmedo utility for dev containers -RUN pip3 --disable-pip-version-check --no-cache-dir install watchdog[watchmedo] - -ENTRYPOINT [ "/docker-init.sh" ] diff --git a/dev/coverage-action/package-lock.json b/dev/coverage-action/package-lock.json index f23a9b66c7..09570ee0e4 100644 --- a/dev/coverage-action/package-lock.json +++ b/dev/coverage-action/package-lock.json @@ -10,9 +10,9 @@ "license": "BSD-3-Clause", "dependencies": { "@actions/core": "1.11.1", - "@actions/github": "6.0.0", + "@actions/github": "6.0.1", "lodash": "4.17.21", - "luxon": "3.5.0" + "luxon": "3.7.1" } }, "node_modules/@actions/core": { @@ -33,14 +33,18 @@ } }, "node_modules/@actions/github": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/@actions/github/-/github-6.0.0.tgz", - "integrity": "sha512-alScpSVnYmjNEXboZjarjukQEzgCRmjMv6Xj47fsdnqGS73bjJNDpiiXmp8jr0UZLdUB6d9jW63IcmddUP+l0g==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/@actions/github/-/github-6.0.1.tgz", + "integrity": "sha512-xbZVcaqD4XnQAe35qSQqskb3SqIAfRyLBrHMd/8TuL7hJSz2QtbDwnNM8zWx4zO5l2fnGtseNE3MbEvD7BxVMw==", + "license": "MIT", "dependencies": { "@actions/http-client": "^2.2.0", "@octokit/core": "^5.0.1", - "@octokit/plugin-paginate-rest": "^9.0.0", - "@octokit/plugin-rest-endpoint-methods": "^10.0.0" + "@octokit/plugin-paginate-rest": "^9.2.2", + "@octokit/plugin-rest-endpoint-methods": "^10.4.0", + "@octokit/request": "^8.4.1", + "@octokit/request-error": "^5.1.1", + "undici": "^5.28.5" } }, "node_modules/@actions/http-client": { @@ -69,20 +73,22 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/@octokit/auth-token/-/auth-token-4.0.0.tgz", "integrity": "sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA==", + "license": "MIT", "engines": { "node": ">= 18" } }, "node_modules/@octokit/core": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.0.1.tgz", - "integrity": "sha512-lyeeeZyESFo+ffI801SaBKmCfsvarO+dgV8/0gD8u1d87clbEdWsP5yC+dSj3zLhb2eIf5SJrn6vDz9AheETHw==", + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.2.1.tgz", + "integrity": "sha512-dKYCMuPO1bmrpuogcjQ8z7ICCH3FP6WmxpwC03yjzGfZhj9fTJg6+bS1+UAplekbN2C+M61UNllGOOoAfGCrdQ==", + "license": "MIT", "dependencies": { "@octokit/auth-token": "^4.0.0", - "@octokit/graphql": "^7.0.0", - "@octokit/request": "^8.0.2", - "@octokit/request-error": "^5.0.0", - "@octokit/types": "^12.0.0", + "@octokit/graphql": "^7.1.0", + "@octokit/request": "^8.4.1", + "@octokit/request-error": "^5.1.1", + "@octokit/types": "^13.0.0", "before-after-hook": "^2.2.0", "universal-user-agent": "^6.0.0" }, @@ -91,12 +97,12 @@ } }, "node_modules/@octokit/endpoint": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.1.tgz", - "integrity": "sha512-hRlOKAovtINHQPYHZlfyFwaM8OyetxeoC81lAkBy34uLb8exrZB50SQdeW3EROqiY9G9yxQTpp5OHTV54QD+vA==", + "version": "9.0.6", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.6.tgz", + "integrity": "sha512-H1fNTMA57HbkFESSt3Y9+FBICv+0jFceJFPWDePYlR/iMGrwM5ph+Dd4XRQs+8X+PUFURLQgX9ChPfhJ/1uNQw==", + "license": "MIT", "dependencies": { - "@octokit/types": "^12.0.0", - "is-plain-object": "^5.0.0", + "@octokit/types": "^13.1.0", "universal-user-agent": "^6.0.0" }, "engines": { @@ -104,12 +110,13 @@ } }, "node_modules/@octokit/graphql": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-7.0.2.tgz", - "integrity": "sha512-OJ2iGMtj5Tg3s6RaXH22cJcxXRi7Y3EBqbHTBRq+PQAqfaS8f/236fUrWhfSn8P4jovyzqucxme7/vWSSZBX2Q==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-7.1.1.tgz", + "integrity": "sha512-3mkDltSfcDUoa176nlGoA32RGjeWjl3K7F/BwHwRMJUW/IteSa4bnSV8p2ThNkcIcZU2umkZWxwETSSCJf2Q7g==", + "license": "MIT", "dependencies": { - "@octokit/request": "^8.0.1", - "@octokit/types": "^12.0.0", + "@octokit/request": "^8.4.1", + "@octokit/types": "^13.0.0", "universal-user-agent": "^6.0.0" }, "engines": { @@ -117,47 +124,80 @@ } }, "node_modules/@octokit/openapi-types": { - "version": "19.0.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-19.0.0.tgz", - "integrity": "sha512-PclQ6JGMTE9iUStpzMkwLCISFn/wDeRjkZFIKALpvJQNBGwDoYYi2fFvuHwssoQ1rXI5mfh6jgTgWuddeUzfWw==" + "version": "24.2.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-24.2.0.tgz", + "integrity": "sha512-9sIH3nSUttelJSXUrmGzl7QUBFul0/mB8HRYl3fOlgHbIWG+WnYDXU3v/2zMtAvuzZ/ed00Ei6on975FhBfzrg==", + "license": "MIT" }, "node_modules/@octokit/plugin-paginate-rest": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-9.0.0.tgz", - "integrity": "sha512-oIJzCpttmBTlEhBmRvb+b9rlnGpmFgDtZ0bB6nq39qIod6A5DP+7RkVLMOixIgRCYSHDTeayWqmiJ2SZ6xgfdw==", + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-9.2.2.tgz", + "integrity": "sha512-u3KYkGF7GcZnSD/3UP0S7K5XUFT2FkOQdcfXZGZQPGv3lm4F2Xbf71lvjldr8c1H3nNbF+33cLEkWYbokGWqiQ==", + "license": "MIT", "dependencies": { - "@octokit/types": "^12.0.0" + "@octokit/types": "^12.6.0" }, "engines": { "node": ">= 18" }, "peerDependencies": { - "@octokit/core": ">=5" + "@octokit/core": "5" + } + }, + "node_modules/@octokit/plugin-paginate-rest/node_modules/@octokit/openapi-types": { + "version": "20.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-20.0.0.tgz", + "integrity": "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA==", + "license": "MIT" + }, + "node_modules/@octokit/plugin-paginate-rest/node_modules/@octokit/types": { + "version": "12.6.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.6.0.tgz", + "integrity": "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==", + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^20.0.0" } }, "node_modules/@octokit/plugin-rest-endpoint-methods": { - "version": "10.0.1", - "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-10.0.1.tgz", - "integrity": "sha512-fgS6HPkPvJiz8CCliewLyym9qAx0RZ/LKh3sATaPfM41y/O2wQ4Z9MrdYeGPVh04wYmHFmWiGlKPC7jWVtZXQA==", + "version": "10.4.1", + "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-10.4.1.tgz", + "integrity": "sha512-xV1b+ceKV9KytQe3zCVqjg+8GTGfDYwaT1ATU5isiUyVtlVAO3HNdzpS4sr4GBx4hxQ46s7ITtZrAsxG22+rVg==", + "license": "MIT", "dependencies": { - "@octokit/types": "^12.0.0" + "@octokit/types": "^12.6.0" }, "engines": { "node": ">= 18" }, "peerDependencies": { - "@octokit/core": ">=5" + "@octokit/core": "5" + } + }, + "node_modules/@octokit/plugin-rest-endpoint-methods/node_modules/@octokit/openapi-types": { + "version": "20.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-20.0.0.tgz", + "integrity": "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA==", + "license": "MIT" + }, + "node_modules/@octokit/plugin-rest-endpoint-methods/node_modules/@octokit/types": { + "version": "12.6.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.6.0.tgz", + "integrity": "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==", + "license": "MIT", + "dependencies": { + "@octokit/openapi-types": "^20.0.0" } }, "node_modules/@octokit/request": { - "version": "8.1.4", - "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.1.4.tgz", - "integrity": "sha512-M0aaFfpGPEKrg7XoA/gwgRvc9MSXHRO2Ioki1qrPDbl1e9YhjIwVoHE7HIKmv/m3idzldj//xBujcFNqGX6ENA==", + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.4.1.tgz", + "integrity": "sha512-qnB2+SY3hkCmBxZsR/MPCybNmbJe4KAlfWErXq+rBKkQJlbjdJeS85VI9r8UqeLYLvnAenU8Q1okM/0MBsAGXw==", + "license": "MIT", "dependencies": { - "@octokit/endpoint": "^9.0.0", - "@octokit/request-error": "^5.0.0", - "@octokit/types": "^12.0.0", - "is-plain-object": "^5.0.0", + "@octokit/endpoint": "^9.0.6", + "@octokit/request-error": "^5.1.1", + "@octokit/types": "^13.1.0", "universal-user-agent": "^6.0.0" }, "engines": { @@ -165,11 +205,12 @@ } }, "node_modules/@octokit/request-error": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.0.1.tgz", - "integrity": "sha512-X7pnyTMV7MgtGmiXBwmO6M5kIPrntOXdyKZLigNfQWSEQzVxR4a4vo49vJjTWX70mPndj8KhfT4Dx+2Ng3vnBQ==", + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.1.1.tgz", + "integrity": "sha512-v9iyEQJH6ZntoENr9/yXxjuezh4My67CBSu9r6Ve/05Iu5gNgnisNWOsoJHTP6k0Rr0+HQIpnH+kyammu90q/g==", + "license": "MIT", "dependencies": { - "@octokit/types": "^12.0.0", + "@octokit/types": "^13.1.0", "deprecation": "^2.0.0", "once": "^1.4.0" }, @@ -178,30 +219,25 @@ } }, "node_modules/@octokit/types": { - "version": "12.0.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.0.0.tgz", - "integrity": "sha512-EzD434aHTFifGudYAygnFlS1Tl6KhbTynEWELQXIbTY8Msvb5nEqTZIm7sbPEt4mQYLZwu3zPKVdeIrw0g7ovg==", + "version": "13.10.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.10.0.tgz", + "integrity": "sha512-ifLaO34EbbPj0Xgro4G5lP5asESjwHracYJvVaPIyXMuiuXLlhic3S47cBdTb+jfODkTE5YtGCLt3Ay3+J97sA==", + "license": "MIT", "dependencies": { - "@octokit/openapi-types": "^19.0.0" + "@octokit/openapi-types": "^24.2.0" } }, "node_modules/before-after-hook": { "version": "2.2.3", "resolved": "https://registry.npmjs.org/before-after-hook/-/before-after-hook-2.2.3.tgz", - "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==" + "integrity": "sha512-NzUnlZexiaH/46WDhANlyR2bXRopNg4F/zuSA3OpZnllCUgRaOF2znDioDWrmbNVsuZk6l9pMquQB38cfBZwkQ==", + "license": "Apache-2.0" }, "node_modules/deprecation": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz", - "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==" - }, - "node_modules/is-plain-object": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", - "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==", - "engines": { - "node": ">=0.10.0" - } + "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==", + "license": "ISC" }, "node_modules/lodash": { "version": "4.17.21", @@ -209,9 +245,9 @@ "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "node_modules/luxon": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz", - "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==", + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.1.tgz", + "integrity": "sha512-RkRWjA926cTvz5rAb1BqyWkKbbjzCGchDUIKMCUvNi17j6f6j8uHGDV82Aqcqtzd+icoYpELmG3ksgGiFNNcNg==", "engines": { "node": ">=12" } @@ -219,7 +255,8 @@ "node_modules/once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", "dependencies": { "wrappy": "1" } @@ -233,9 +270,10 @@ } }, "node_modules/undici": { - "version": "5.26.4", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.26.4.tgz", - "integrity": "sha512-OG+QOf0fTLtazL9P9X7yqWxQ+Z0395Wk6DSkyTxtaq3wQEjIroVe7Y4asCX/vcCxYpNGMnwz8F0qbRYUoaQVMw==", + "version": "5.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.29.0.tgz", + "integrity": "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==", + "license": "MIT", "dependencies": { "@fastify/busboy": "^2.0.0" }, @@ -244,14 +282,16 @@ } }, "node_modules/universal-user-agent": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz", - "integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w==" + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.1.tgz", + "integrity": "sha512-yCzhz6FN2wU1NiiQRogkTQszlQSlpWaw8SvVegAc+bDxbzHgh1vX8uIe8OYyMH6DwH+sdTJsgMl36+mSMdRJIQ==", + "license": "ISC" }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" } }, "dependencies": { @@ -273,14 +313,17 @@ } }, "@actions/github": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/@actions/github/-/github-6.0.0.tgz", - "integrity": "sha512-alScpSVnYmjNEXboZjarjukQEzgCRmjMv6Xj47fsdnqGS73bjJNDpiiXmp8jr0UZLdUB6d9jW63IcmddUP+l0g==", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/@actions/github/-/github-6.0.1.tgz", + "integrity": "sha512-xbZVcaqD4XnQAe35qSQqskb3SqIAfRyLBrHMd/8TuL7hJSz2QtbDwnNM8zWx4zO5l2fnGtseNE3MbEvD7BxVMw==", "requires": { "@actions/http-client": "^2.2.0", "@octokit/core": "^5.0.1", - "@octokit/plugin-paginate-rest": "^9.0.0", - "@octokit/plugin-rest-endpoint-methods": "^10.0.0" + "@octokit/plugin-paginate-rest": "^9.2.2", + "@octokit/plugin-rest-endpoint-methods": "^10.4.0", + "@octokit/request": "^8.4.1", + "@octokit/request-error": "^5.1.1", + "undici": "^5.28.5" } }, "@actions/http-client": { @@ -308,88 +351,116 @@ "integrity": "sha512-tY/msAuJo6ARbK6SPIxZrPBms3xPbfwBrulZe0Wtr/DIY9lje2HeV1uoebShn6mx7SjCHif6EjMvoREj+gZ+SA==" }, "@octokit/core": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.0.1.tgz", - "integrity": "sha512-lyeeeZyESFo+ffI801SaBKmCfsvarO+dgV8/0gD8u1d87clbEdWsP5yC+dSj3zLhb2eIf5SJrn6vDz9AheETHw==", + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/@octokit/core/-/core-5.2.1.tgz", + "integrity": "sha512-dKYCMuPO1bmrpuogcjQ8z7ICCH3FP6WmxpwC03yjzGfZhj9fTJg6+bS1+UAplekbN2C+M61UNllGOOoAfGCrdQ==", "requires": { "@octokit/auth-token": "^4.0.0", - "@octokit/graphql": "^7.0.0", - "@octokit/request": "^8.0.2", - "@octokit/request-error": "^5.0.0", - "@octokit/types": "^12.0.0", + "@octokit/graphql": "^7.1.0", + "@octokit/request": "^8.4.1", + "@octokit/request-error": "^5.1.1", + "@octokit/types": "^13.0.0", "before-after-hook": "^2.2.0", "universal-user-agent": "^6.0.0" } }, "@octokit/endpoint": { - "version": "9.0.1", - "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.1.tgz", - "integrity": "sha512-hRlOKAovtINHQPYHZlfyFwaM8OyetxeoC81lAkBy34uLb8exrZB50SQdeW3EROqiY9G9yxQTpp5OHTV54QD+vA==", + "version": "9.0.6", + "resolved": "https://registry.npmjs.org/@octokit/endpoint/-/endpoint-9.0.6.tgz", + "integrity": "sha512-H1fNTMA57HbkFESSt3Y9+FBICv+0jFceJFPWDePYlR/iMGrwM5ph+Dd4XRQs+8X+PUFURLQgX9ChPfhJ/1uNQw==", "requires": { - "@octokit/types": "^12.0.0", - "is-plain-object": "^5.0.0", + "@octokit/types": "^13.1.0", "universal-user-agent": "^6.0.0" } }, "@octokit/graphql": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-7.0.2.tgz", - "integrity": "sha512-OJ2iGMtj5Tg3s6RaXH22cJcxXRi7Y3EBqbHTBRq+PQAqfaS8f/236fUrWhfSn8P4jovyzqucxme7/vWSSZBX2Q==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/@octokit/graphql/-/graphql-7.1.1.tgz", + "integrity": "sha512-3mkDltSfcDUoa176nlGoA32RGjeWjl3K7F/BwHwRMJUW/IteSa4bnSV8p2ThNkcIcZU2umkZWxwETSSCJf2Q7g==", "requires": { - "@octokit/request": "^8.0.1", - "@octokit/types": "^12.0.0", + "@octokit/request": "^8.4.1", + "@octokit/types": "^13.0.0", "universal-user-agent": "^6.0.0" } }, "@octokit/openapi-types": { - "version": "19.0.0", - "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-19.0.0.tgz", - "integrity": "sha512-PclQ6JGMTE9iUStpzMkwLCISFn/wDeRjkZFIKALpvJQNBGwDoYYi2fFvuHwssoQ1rXI5mfh6jgTgWuddeUzfWw==" + "version": "24.2.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-24.2.0.tgz", + "integrity": "sha512-9sIH3nSUttelJSXUrmGzl7QUBFul0/mB8HRYl3fOlgHbIWG+WnYDXU3v/2zMtAvuzZ/ed00Ei6on975FhBfzrg==" }, "@octokit/plugin-paginate-rest": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-9.0.0.tgz", - "integrity": "sha512-oIJzCpttmBTlEhBmRvb+b9rlnGpmFgDtZ0bB6nq39qIod6A5DP+7RkVLMOixIgRCYSHDTeayWqmiJ2SZ6xgfdw==", + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/@octokit/plugin-paginate-rest/-/plugin-paginate-rest-9.2.2.tgz", + "integrity": "sha512-u3KYkGF7GcZnSD/3UP0S7K5XUFT2FkOQdcfXZGZQPGv3lm4F2Xbf71lvjldr8c1H3nNbF+33cLEkWYbokGWqiQ==", "requires": { - "@octokit/types": "^12.0.0" + "@octokit/types": "^12.6.0" + }, + "dependencies": { + "@octokit/openapi-types": { + "version": "20.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-20.0.0.tgz", + "integrity": "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA==" + }, + "@octokit/types": { + "version": "12.6.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.6.0.tgz", + "integrity": "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==", + "requires": { + "@octokit/openapi-types": "^20.0.0" + } + } } }, "@octokit/plugin-rest-endpoint-methods": { - "version": "10.0.1", - "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-10.0.1.tgz", - "integrity": "sha512-fgS6HPkPvJiz8CCliewLyym9qAx0RZ/LKh3sATaPfM41y/O2wQ4Z9MrdYeGPVh04wYmHFmWiGlKPC7jWVtZXQA==", + "version": "10.4.1", + "resolved": "https://registry.npmjs.org/@octokit/plugin-rest-endpoint-methods/-/plugin-rest-endpoint-methods-10.4.1.tgz", + "integrity": "sha512-xV1b+ceKV9KytQe3zCVqjg+8GTGfDYwaT1ATU5isiUyVtlVAO3HNdzpS4sr4GBx4hxQ46s7ITtZrAsxG22+rVg==", "requires": { - "@octokit/types": "^12.0.0" + "@octokit/types": "^12.6.0" + }, + "dependencies": { + "@octokit/openapi-types": { + "version": "20.0.0", + "resolved": "https://registry.npmjs.org/@octokit/openapi-types/-/openapi-types-20.0.0.tgz", + "integrity": "sha512-EtqRBEjp1dL/15V7WiX5LJMIxxkdiGJnabzYx5Apx4FkQIFgAfKumXeYAqqJCj1s+BMX4cPFIFC4OLCR6stlnA==" + }, + "@octokit/types": { + "version": "12.6.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.6.0.tgz", + "integrity": "sha512-1rhSOfRa6H9w4YwK0yrf5faDaDTb+yLyBUKOCV4xtCDB5VmIPqd/v9yr9o6SAzOAlRxMiRiCic6JVM1/kunVkw==", + "requires": { + "@octokit/openapi-types": "^20.0.0" + } + } } }, "@octokit/request": { - "version": "8.1.4", - "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.1.4.tgz", - "integrity": "sha512-M0aaFfpGPEKrg7XoA/gwgRvc9MSXHRO2Ioki1qrPDbl1e9YhjIwVoHE7HIKmv/m3idzldj//xBujcFNqGX6ENA==", + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/@octokit/request/-/request-8.4.1.tgz", + "integrity": "sha512-qnB2+SY3hkCmBxZsR/MPCybNmbJe4KAlfWErXq+rBKkQJlbjdJeS85VI9r8UqeLYLvnAenU8Q1okM/0MBsAGXw==", "requires": { - "@octokit/endpoint": "^9.0.0", - "@octokit/request-error": "^5.0.0", - "@octokit/types": "^12.0.0", - "is-plain-object": "^5.0.0", + "@octokit/endpoint": "^9.0.6", + "@octokit/request-error": "^5.1.1", + "@octokit/types": "^13.1.0", "universal-user-agent": "^6.0.0" } }, "@octokit/request-error": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.0.1.tgz", - "integrity": "sha512-X7pnyTMV7MgtGmiXBwmO6M5kIPrntOXdyKZLigNfQWSEQzVxR4a4vo49vJjTWX70mPndj8KhfT4Dx+2Ng3vnBQ==", + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/@octokit/request-error/-/request-error-5.1.1.tgz", + "integrity": "sha512-v9iyEQJH6ZntoENr9/yXxjuezh4My67CBSu9r6Ve/05Iu5gNgnisNWOsoJHTP6k0Rr0+HQIpnH+kyammu90q/g==", "requires": { - "@octokit/types": "^12.0.0", + "@octokit/types": "^13.1.0", "deprecation": "^2.0.0", "once": "^1.4.0" } }, "@octokit/types": { - "version": "12.0.0", - "resolved": "https://registry.npmjs.org/@octokit/types/-/types-12.0.0.tgz", - "integrity": "sha512-EzD434aHTFifGudYAygnFlS1Tl6KhbTynEWELQXIbTY8Msvb5nEqTZIm7sbPEt4mQYLZwu3zPKVdeIrw0g7ovg==", + "version": "13.10.0", + "resolved": "https://registry.npmjs.org/@octokit/types/-/types-13.10.0.tgz", + "integrity": "sha512-ifLaO34EbbPj0Xgro4G5lP5asESjwHracYJvVaPIyXMuiuXLlhic3S47cBdTb+jfODkTE5YtGCLt3Ay3+J97sA==", "requires": { - "@octokit/openapi-types": "^19.0.0" + "@octokit/openapi-types": "^24.2.0" } }, "before-after-hook": { @@ -402,25 +473,20 @@ "resolved": "https://registry.npmjs.org/deprecation/-/deprecation-2.3.1.tgz", "integrity": "sha512-xmHIy4F3scKVwMsQ4WnVaS8bHOx0DmVwRywosKhaILI0ywMDWPtBSku2HNxRvF7jtwDRsoEwYQSfbxj8b7RlJQ==" }, - "is-plain-object": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-5.0.0.tgz", - "integrity": "sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==" - }, "lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "luxon": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz", - "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==" + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.1.tgz", + "integrity": "sha512-RkRWjA926cTvz5rAb1BqyWkKbbjzCGchDUIKMCUvNi17j6f6j8uHGDV82Aqcqtzd+icoYpELmG3ksgGiFNNcNg==" }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", - "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", "requires": { "wrappy": "1" } @@ -431,22 +497,22 @@ "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==" }, "undici": { - "version": "5.26.4", - "resolved": "https://registry.npmjs.org/undici/-/undici-5.26.4.tgz", - "integrity": "sha512-OG+QOf0fTLtazL9P9X7yqWxQ+Z0395Wk6DSkyTxtaq3wQEjIroVe7Y4asCX/vcCxYpNGMnwz8F0qbRYUoaQVMw==", + "version": "5.29.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-5.29.0.tgz", + "integrity": "sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==", "requires": { "@fastify/busboy": "^2.0.0" } }, "universal-user-agent": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.0.tgz", - "integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w==" + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/universal-user-agent/-/universal-user-agent-6.0.1.tgz", + "integrity": "sha512-yCzhz6FN2wU1NiiQRogkTQszlQSlpWaw8SvVegAc+bDxbzHgh1vX8uIe8OYyMH6DwH+sdTJsgMl36+mSMdRJIQ==" }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", - "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==" } } } diff --git a/dev/coverage-action/package.json b/dev/coverage-action/package.json index 1d03fe3dab..3f72b78028 100644 --- a/dev/coverage-action/package.json +++ b/dev/coverage-action/package.json @@ -7,8 +7,8 @@ "license": "BSD-3-Clause", "dependencies": { "@actions/core": "1.11.1", - "@actions/github": "6.0.0", + "@actions/github": "6.0.1", "lodash": "4.17.21", - "luxon": "3.5.0" + "luxon": "3.7.1" } } diff --git a/dev/deploy-to-container/cli.js b/dev/deploy-to-container/cli.js index 4aee7ba1ed..1a2d993ac4 100644 --- a/dev/deploy-to-container/cli.js +++ b/dev/deploy-to-container/cli.js @@ -70,6 +70,7 @@ async function main () { .replace('__HOSTNAME__', hostname) ) await fs.copy(path.join(basePath, 'docker/scripts/app-create-dirs.sh'), path.join(releasePath, 'app-create-dirs.sh')) + await fs.copy(path.join(basePath, 'docker/scripts/app-init-celery.sh'), path.join(releasePath, 'app-init-celery.sh')) await fs.copy(path.join(basePath, 'dev/deploy-to-container/start.sh'), path.join(releasePath, 'start.sh')) await fs.copy(path.join(basePath, 'test/data'), path.join(releasePath, 'test/data')) console.info('Updated configuration files.') @@ -98,14 +99,6 @@ async function main () { }) console.info('Pulled latest MQ docker image.') - // Pull latest Celery image - console.info('Pulling latest Celery docker image...') - const celeryImagePullStream = await dock.pull('ghcr.io/ietf-tools/datatracker-celery:latest') - await new Promise((resolve, reject) => { - dock.modem.followProgress(celeryImagePullStream, (err, res) => err ? reject(err) : resolve(res)) - }) - console.info('Pulled latest Celery docker image.') - // Terminate existing containers console.info('Ensuring existing containers with same name are terminated...') const containers = await dock.listContainers({ all: true }) @@ -221,7 +214,7 @@ async function main () { const celeryContainers = {} for (const conConf of conConfs) { celeryContainers[conConf.name] = await dock.createContainer({ - Image: 'ghcr.io/ietf-tools/datatracker-celery:latest', + Image: 'ghcr.io/ietf-tools/datatracker-app-base:latest', name: `dt-${conConf.name}-${branch}`, Hostname: `dt-${conConf.name}-${branch}`, Env: [ @@ -243,7 +236,7 @@ async function main () { Name: 'unless-stopped' } }, - Cmd: ['--loglevel=INFO'] + Entrypoint: ['bash', '-c', 'chmod +x ./app-init-celery.sh && ./app-init-celery.sh'] }) } console.info('Created Celery docker containers successfully.') diff --git a/dev/deploy-to-container/package-lock.json b/dev/deploy-to-container/package-lock.json index f4ce2c367c..5d5bef5604 100644 --- a/dev/deploy-to-container/package-lock.json +++ b/dev/deploy-to-container/package-lock.json @@ -6,12 +6,12 @@ "": { "name": "deploy-to-container", "dependencies": { - "dockerode": "^4.0.4", - "fs-extra": "^11.3.0", - "nanoid": "5.1.4", - "nanoid-dictionary": "5.0.0-beta.1", - "slugify": "1.6.6", - "tar": "^7.4.3", + "dockerode": "^4.0.10", + "fs-extra": "^11.3.4", + "nanoid": "5.1.7", + "nanoid-dictionary": "5.0.0", + "slugify": "1.6.9", + "tar": "^7.5.13", "yargs": "^17.7.2" }, "engines": { @@ -52,95 +52,6 @@ "node": ">=6" } }, - "node_modules/@isaacs/cliui": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", - "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", - "dependencies": { - "string-width": "^5.1.2", - "string-width-cjs": "npm:string-width@^4.2.0", - "strip-ansi": "^7.0.1", - "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", - "wrap-ansi": "^8.1.0", - "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/@isaacs/cliui/node_modules/ansi-regex": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", - "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-regex?sponsor=1" - } - }, - "node_modules/@isaacs/cliui/node_modules/ansi-styles": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", - "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/@isaacs/cliui/node_modules/emoji-regex": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" - }, - "node_modules/@isaacs/cliui/node_modules/string-width": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", - "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", - "dependencies": { - "eastasianwidth": "^0.2.0", - "emoji-regex": "^9.2.2", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/@isaacs/cliui/node_modules/strip-ansi": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", - "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", - "dependencies": { - "ansi-regex": "^6.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/strip-ansi?sponsor=1" - } - }, - "node_modules/@isaacs/cliui/node_modules/wrap-ansi": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", - "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", - "dependencies": { - "ansi-styles": "^6.1.0", - "string-width": "^5.0.1", - "strip-ansi": "^7.0.1" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, "node_modules/@isaacs/fs-minipass": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", @@ -161,15 +72,6 @@ "url": "https://opencollective.com/js-sdsl" } }, - "node_modules/@pkgjs/parseargs": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", - "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", - "optional": true, - "engines": { - "node": ">=14" - } - }, "node_modules/@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -258,16 +160,10 @@ "version": "0.2.6", "resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.6.tgz", "integrity": "sha512-ix/FxPn0MDjeyJ7i/yoHGFt/EX6LyNbxSEhPPXODPL+KB0VPk86UYfL0lMdy+KCnv+fmvIzySwaK5COwqVbWTQ==", - "license": "MIT", "dependencies": { "safer-buffer": "~2.1.0" } }, - "node_modules/balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" - }, "node_modules/base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -291,23 +187,47 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.2.tgz", "integrity": "sha512-qeFIXtP4MSoi6NLqO12WfqARWWuCKi2Rn/9hJLEmtB5yTNr9DqFWkJRCf2qShWzPeAMRnOgCrq0sg/KLv5ES9w==", - "license": "BSD-3-Clause", "dependencies": { "tweetnacl": "^0.14.3" } }, - "node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", "dependencies": { - "balanced-match": "^1.0.0" + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" } }, "node_modules/buildcheck": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", - "integrity": "sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==", + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.7.tgz", + "integrity": "sha512-lHblz4ahamxpTmnsk+MNTRWsjYKv965MwOrSJyeD588rR3Jcu7swE+0wN5F+PbL5cjgu/9ObkhfzEPuofEMwLA==", "optional": true, "engines": { "node": ">=10.0.0" @@ -361,24 +281,10 @@ "node": ">=10.0.0" } }, - "node_modules/cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", - "dependencies": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - }, - "engines": { - "node": ">= 8" - } - }, "node_modules/debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", - "license": "MIT", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "dependencies": { "ms": "^2.1.3" }, @@ -392,10 +298,9 @@ } }, "node_modules/docker-modem": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.6.tgz", - "integrity": "sha512-ens7BiayssQz/uAxGzH8zGXCtiV24rRWXdjNha5V4zSOcxmAZsfGVm/PPFbwQdqEkDnhG+SyR9E3zSHUbOKXBQ==", - "license": "Apache-2.0", + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.7.tgz", + "integrity": "sha512-XJgGhoR/CLpqshm4d3L7rzH6t8NgDFUIIpztYlLHIApeJjMZKYJMz2zxPsYxnejq5h3ELYSw/RBsi3t5h7gNTA==", "dependencies": { "debug": "^4.1.1", "readable-stream": "^3.5.0", @@ -407,37 +312,31 @@ } }, "node_modules/dockerode": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.4.tgz", - "integrity": "sha512-6GYP/EdzEY50HaOxTVTJ2p+mB5xDHTMJhS+UoGrVyS6VC+iQRh7kZ4FRpUYq6nziby7hPqWhOrFFUFTMUZJJ5w==", - "license": "Apache-2.0", + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.10.tgz", + "integrity": "sha512-8L/P9JynLBiG7/coiA4FlQXegHltRqS0a+KqI44P1zgQh8QLHTg7FKOwhkBgSJwZTeHsq30WRoVFLuwkfK0YFg==", "dependencies": { "@balena/dockerignore": "^1.0.2", "@grpc/grpc-js": "^1.11.1", "@grpc/proto-loader": "^0.7.13", - "docker-modem": "^5.0.6", + "docker-modem": "^5.0.7", "protobufjs": "^7.3.2", - "tar-fs": "~2.0.1", + "tar-fs": "^2.1.4", "uuid": "^10.0.0" }, "engines": { "node": ">= 8.0" } }, - "node_modules/eastasianwidth": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", - "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" - }, "node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, "node_modules/end-of-stream": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", - "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", "dependencies": { "once": "^1.4.0" } @@ -450,31 +349,15 @@ "node": ">=6" } }, - "node_modules/foreground-child": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", - "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", - "dependencies": { - "cross-spawn": "^7.0.0", - "signal-exit": "^4.0.1" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" }, "node_modules/fs-extra": { - "version": "11.3.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.0.tgz", - "integrity": "sha512-Z4XaCL6dUDHfP/jT25jJKMmtxvuwbkrD1vNSMFlo9lNLY2c5FHYSQgHPRZUjAB26TpDEoW9HCOgplrdbaPV/ew==", - "license": "MIT", + "version": "11.3.4", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.4.tgz", + "integrity": "sha512-CTXd6rk/M3/ULNQj8FBqBWHYBVYybQ3VPBw0xGKFe3tuH7ytT6ACnvzpIQ3UZtB8yvUKC2cXn1a+x+5EVQLovA==", "dependencies": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", @@ -492,27 +375,6 @@ "node": "6.* || 8.* || >= 10.*" } }, - "node_modules/glob": { - "version": "10.3.12", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.12.tgz", - "integrity": "sha512-TCNv8vJ+xz4QiqTpfOJA7HvYv+tNIRHKfUWw/q+v2jdgN4ebz+KY9tGx5J4rHP0o84mNP+ApH66HRX8us3Khqg==", - "dependencies": { - "foreground-child": "^3.1.0", - "jackspeak": "^2.3.6", - "minimatch": "^9.0.1", - "minipass": "^7.0.4", - "path-scurry": "^1.10.2" - }, - "bin": { - "glob": "dist/esm/bin.mjs" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/graceful-fs": { "version": "4.2.10", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", @@ -550,28 +412,6 @@ "node": ">=8" } }, - "node_modules/isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" - }, - "node_modules/jackspeak": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", - "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", - "dependencies": { - "@isaacs/cliui": "^8.0.2" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - }, - "optionalDependencies": { - "@pkgjs/parseargs": "^0.11.0" - } - }, "node_modules/jsonfile": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", @@ -593,28 +433,6 @@ "resolved": "https://registry.npmjs.org/long/-/long-5.2.4.tgz", "integrity": "sha512-qtzLbJE8hq7VabR3mISmVGtoXP8KGc2Z/AT8OuqlYD7JTR3oqrgwdjnk07wpj1twXxYmgDXgoKVWUG/fReSzHg==" }, - "node_modules/lru-cache": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.2.tgz", - "integrity": "sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==", - "engines": { - "node": "14 || >=16.14" - } - }, - "node_modules/minimatch": { - "version": "9.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", - "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/minipass": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", @@ -624,31 +442,16 @@ } }, "node_modules/minizlib": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz", - "integrity": "sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", + "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", "dependencies": { - "minipass": "^7.0.4", - "rimraf": "^5.0.5" + "minipass": "^7.1.2" }, "engines": { "node": ">= 18" } }, - "node_modules/mkdirp": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", - "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==", - "bin": { - "mkdirp": "dist/cjs/src/bin.js" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/mkdirp-classic": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", @@ -657,27 +460,24 @@ "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", - "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", - "license": "MIT" + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "node_modules/nan": { - "version": "2.22.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.0.tgz", - "integrity": "sha512-nbajikzWTMwsW+eSsNm3QwlOs7het9gGJU5dDZzRTQGk03vyBOauxgI4VakDzE0PtsGTmXPsXTbbjVhRwR5mpw==", - "license": "MIT", + "version": "2.26.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.26.2.tgz", + "integrity": "sha512-0tTvBTYkt3tdGw22nrAy50x7gpbGCCFH3AFcyS5WiUu7Eu4vWlri1woE6qHBSfy11vksDqkiwjOnlR7WV8G1Hw==", "optional": true }, "node_modules/nanoid": { - "version": "5.1.4", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.4.tgz", - "integrity": "sha512-GTFcMIDgR7tqji/LpSY8rtg464VnJl/j6ypoehYnuGb+Y8qZUdtKB8WVCXon0UEZgFDbuUxpIl//6FHLHgXSNA==", + "version": "5.1.7", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.7.tgz", + "integrity": "sha512-ua3NDgISf6jdwezAheMOk4mbE1LXjm1DfMUDMuJf4AqxLFK3ccGpgWizwa5YV7Yz9EpXwEaWoRXSb/BnV0t5dQ==", "funding": [ { "type": "github", "url": "https://github.com/sponsors/ai" } ], - "license": "MIT", "bin": { "nanoid": "bin/nanoid.js" }, @@ -686,9 +486,10 @@ } }, "node_modules/nanoid-dictionary": { - "version": "5.0.0-beta.1", - "resolved": "https://registry.npmjs.org/nanoid-dictionary/-/nanoid-dictionary-5.0.0-beta.1.tgz", - "integrity": "sha512-xBkL9zzkNjzJ/UnmWyiOUDVX/COoi05eS0oU28RYKFFQhdnzO5dTOPbVZ/fCFgIOGr1zNinDHJ68mm/KQfcgcw==" + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/nanoid-dictionary/-/nanoid-dictionary-5.0.0.tgz", + "integrity": "sha512-/iCyQHwt36XkaIvSE9fcC8p6DiMPCZMTSMj9UT56Cv6T7f5CuxvOMhpNncaNieQ4z4d32p7ruEtAfRsb7Ya8Gw==", + "license": "MIT" }, "node_modules/once": { "version": "1.4.0", @@ -698,29 +499,6 @@ "wrappy": "1" } }, - "node_modules/path-key": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==", - "engines": { - "node": ">=8" - } - }, - "node_modules/path-scurry": { - "version": "1.10.2", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.2.tgz", - "integrity": "sha512-7xTavNy5RQXnsjANvVvMkEjvloOinkAjv/Z6Ildz9v2RinZ4SBKTWFOVRbaF8p0vpHnyjV/UwNDdKuUv6M5qcA==", - "dependencies": { - "lru-cache": "^10.2.0", - "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" - }, - "engines": { - "node": ">=16 || 14 >=14.17" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/protobufjs": { "version": "7.4.0", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.4.0.tgz", @@ -745,9 +523,9 @@ } }, "node_modules/pump": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", - "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", + "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -774,23 +552,6 @@ "node": ">=0.10.0" } }, - "node_modules/rimraf": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.5.tgz", - "integrity": "sha512-CqDakW+hMe/Bz202FPEymy68P+G50RfMQK+Qo5YUqc9SPipvbGjCGKd0RSKEelbsfQuw3g5NZDSrlZZAJurH1A==", - "dependencies": { - "glob": "^10.3.7" - }, - "bin": { - "rimraf": "dist/esm/bin.mjs" - }, - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -813,43 +574,12 @@ "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", - "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", - "license": "MIT" - }, - "node_modules/shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "dependencies": { - "shebang-regex": "^3.0.0" - }, - "engines": { - "node": ">=8" - } - }, - "node_modules/shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==", - "engines": { - "node": ">=8" - } - }, - "node_modules/signal-exit": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", - "engines": { - "node": ">=14" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "node_modules/slugify": { - "version": "1.6.6", - "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.6.tgz", - "integrity": "sha512-h+z7HKHYXj6wJU+AnS/+IH8Uh9fdcX1Lrhg1/VMdf9PwoBQXFcXiAdsy2tSK0P6gKwJLXp02r90ahUCqHk9rrw==", + "version": "1.6.9", + "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.9.tgz", + "integrity": "sha512-vZ7rfeehZui7wQs438JXBckYLkIIdfHOXsaVEUMyS5fHo1483l1bMdo0EDSWYclY0yZKFOipDy4KHuKs6ssvdg==", "engines": { "node": ">=8.0.0" } @@ -857,13 +587,12 @@ "node_modules/split-ca": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/split-ca/-/split-ca-1.0.1.tgz", - "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==", - "license": "ISC" + "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==" }, "node_modules/ssh2": { - "version": "1.16.0", - "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.16.0.tgz", - "integrity": "sha512-r1X4KsBGedJqo7h8F5c4Ybpcr5RjyP+aWIG007uBPRjmdQWfEiVLzSK71Zji1B9sKxwaCvD8y8cwSkYrlLiRRg==", + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.17.0.tgz", + "integrity": "sha512-wPldCk3asibAjQ/kziWQQt1Wh3PgDFpC0XpwclzKcdT1vql6KeYxf5LIt4nlFkUeR8WuphYMKqUA56X4rjbfgQ==", "hasInstallScript": true, "dependencies": { "asn1": "^0.2.6", @@ -874,7 +603,7 @@ }, "optionalDependencies": { "cpu-features": "~0.0.10", - "nan": "^2.20.0" + "nan": "^2.23.0" } }, "node_modules/string_decoder": { @@ -898,20 +627,6 @@ "node": ">=8" } }, - "node_modules/string-width-cjs": { - "name": "string-width", - "version": "4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dependencies": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -923,28 +638,15 @@ "node": ">=8" } }, - "node_modules/strip-ansi-cjs": { - "name": "strip-ansi", - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dependencies": { - "ansi-regex": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/tar": { - "version": "7.4.3", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz", - "integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==", + "version": "7.5.13", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.13.tgz", + "integrity": "sha512-tOG/7GyXpFevhXVh8jOPJrmtRpOTsYqUIkVdVooZYJS/z8WhfQUX8RJILmeuJNinGAMSu1veBr4asSHFt5/hng==", "dependencies": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", "minipass": "^7.1.2", - "minizlib": "^3.0.1", - "mkdirp": "^3.0.1", + "minizlib": "^3.1.0", "yallist": "^5.0.0" }, "engines": { @@ -952,14 +654,14 @@ } }, "node_modules/tar-fs": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.0.1.tgz", - "integrity": "sha512-6tzWDMeroL87uF/+lin46k+Q+46rAJ0SyPGz7OW7wTgblI273hsBqk2C1j0/xNadNLKDTUL9BukSjB7cwgmlPA==", + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz", + "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==", "dependencies": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", "pump": "^3.0.0", - "tar-stream": "^2.0.0" + "tar-stream": "^2.1.4" } }, "node_modules/tar-stream": { @@ -977,39 +679,6 @@ "node": ">=6" } }, - "node_modules/tar-stream/node_modules/bl": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", - "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", - "dependencies": { - "buffer": "^5.5.0", - "inherits": "^2.0.4", - "readable-stream": "^3.4.0" - } - }, - "node_modules/tar-stream/node_modules/buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - }, "node_modules/tar/node_modules/chownr": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", @@ -1021,8 +690,7 @@ "node_modules/tweetnacl": { "version": "0.14.5", "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz", - "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==", - "license": "Unlicense" + "integrity": "sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA==" }, "node_modules/undici-types": { "version": "6.20.0", @@ -1054,20 +722,6 @@ "uuid": "dist/bin/uuid" } }, - "node_modules/which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "node-which": "bin/node-which" - }, - "engines": { - "node": ">= 8" - } - }, "node_modules/wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -1084,23 +738,6 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, - "node_modules/wrap-ansi-cjs": { - "name": "wrap-ansi", - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dependencies": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/wrap-ansi?sponsor=1" - } - }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -1174,64 +811,6 @@ "yargs": "^17.7.2" } }, - "@isaacs/cliui": { - "version": "8.0.2", - "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", - "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", - "requires": { - "string-width": "^5.1.2", - "string-width-cjs": "npm:string-width@^4.2.0", - "strip-ansi": "^7.0.1", - "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", - "wrap-ansi": "^8.1.0", - "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" - }, - "dependencies": { - "ansi-regex": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", - "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==" - }, - "ansi-styles": { - "version": "6.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", - "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==" - }, - "emoji-regex": { - "version": "9.2.2", - "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" - }, - "string-width": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", - "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", - "requires": { - "eastasianwidth": "^0.2.0", - "emoji-regex": "^9.2.2", - "strip-ansi": "^7.0.1" - } - }, - "strip-ansi": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", - "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", - "requires": { - "ansi-regex": "^6.0.1" - } - }, - "wrap-ansi": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", - "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", - "requires": { - "ansi-styles": "^6.1.0", - "string-width": "^5.0.1", - "strip-ansi": "^7.0.1" - } - } - } - }, "@isaacs/fs-minipass": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/@isaacs/fs-minipass/-/fs-minipass-4.0.1.tgz", @@ -1245,12 +824,6 @@ "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==" }, - "@pkgjs/parseargs": { - "version": "0.11.0", - "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", - "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", - "optional": true - }, "@protobufjs/aspromise": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", @@ -1334,11 +907,6 @@ "safer-buffer": "~2.1.0" } }, - "balanced-match": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", - "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" - }, "base64-js": { "version": "1.5.1", "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", @@ -1352,18 +920,29 @@ "tweetnacl": "^0.14.3" } }, - "brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", "requires": { - "balanced-match": "^1.0.0" + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" } }, "buildcheck": { - "version": "0.0.6", - "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.6.tgz", - "integrity": "sha512-8f9ZJCUXyT1M35Jx7MkBgmBMo3oHTTBIPLiY9xyL0pl3T5RwcPEY8cUHr5LBNfu/fk6c2T4DJZuVM/8ZZT2D2A==", + "version": "0.0.7", + "resolved": "https://registry.npmjs.org/buildcheck/-/buildcheck-0.0.7.tgz", + "integrity": "sha512-lHblz4ahamxpTmnsk+MNTRWsjYKv965MwOrSJyeD588rR3Jcu7swE+0wN5F+PbL5cjgu/9ObkhfzEPuofEMwLA==", "optional": true }, "chownr": { @@ -1404,28 +983,18 @@ "nan": "^2.19.0" } }, - "cross-spawn": { - "version": "7.0.3", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", - "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==", - "requires": { - "path-key": "^3.1.0", - "shebang-command": "^2.0.0", - "which": "^2.0.1" - } - }, "debug": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", - "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", "requires": { "ms": "^2.1.3" } }, "docker-modem": { - "version": "5.0.6", - "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.6.tgz", - "integrity": "sha512-ens7BiayssQz/uAxGzH8zGXCtiV24rRWXdjNha5V4zSOcxmAZsfGVm/PPFbwQdqEkDnhG+SyR9E3zSHUbOKXBQ==", + "version": "5.0.7", + "resolved": "https://registry.npmjs.org/docker-modem/-/docker-modem-5.0.7.tgz", + "integrity": "sha512-XJgGhoR/CLpqshm4d3L7rzH6t8NgDFUIIpztYlLHIApeJjMZKYJMz2zxPsYxnejq5h3ELYSw/RBsi3t5h7gNTA==", "requires": { "debug": "^4.1.1", "readable-stream": "^3.5.0", @@ -1434,33 +1003,28 @@ } }, "dockerode": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.4.tgz", - "integrity": "sha512-6GYP/EdzEY50HaOxTVTJ2p+mB5xDHTMJhS+UoGrVyS6VC+iQRh7kZ4FRpUYq6nziby7hPqWhOrFFUFTMUZJJ5w==", + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.10.tgz", + "integrity": "sha512-8L/P9JynLBiG7/coiA4FlQXegHltRqS0a+KqI44P1zgQh8QLHTg7FKOwhkBgSJwZTeHsq30WRoVFLuwkfK0YFg==", "requires": { "@balena/dockerignore": "^1.0.2", "@grpc/grpc-js": "^1.11.1", "@grpc/proto-loader": "^0.7.13", - "docker-modem": "^5.0.6", + "docker-modem": "^5.0.7", "protobufjs": "^7.3.2", - "tar-fs": "~2.0.1", + "tar-fs": "^2.1.4", "uuid": "^10.0.0" } }, - "eastasianwidth": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", - "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" - }, "emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, "end-of-stream": { - "version": "1.4.4", - "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", - "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", "requires": { "once": "^1.4.0" } @@ -1470,24 +1034,15 @@ "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" }, - "foreground-child": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", - "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", - "requires": { - "cross-spawn": "^7.0.0", - "signal-exit": "^4.0.1" - } - }, "fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" }, "fs-extra": { - "version": "11.3.0", - "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.0.tgz", - "integrity": "sha512-Z4XaCL6dUDHfP/jT25jJKMmtxvuwbkrD1vNSMFlo9lNLY2c5FHYSQgHPRZUjAB26TpDEoW9HCOgplrdbaPV/ew==", + "version": "11.3.4", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.4.tgz", + "integrity": "sha512-CTXd6rk/M3/ULNQj8FBqBWHYBVYybQ3VPBw0xGKFe3tuH7ytT6ACnvzpIQ3UZtB8yvUKC2cXn1a+x+5EVQLovA==", "requires": { "graceful-fs": "^4.2.0", "jsonfile": "^6.0.1", @@ -1499,18 +1054,6 @@ "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" }, - "glob": { - "version": "10.3.12", - "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.12.tgz", - "integrity": "sha512-TCNv8vJ+xz4QiqTpfOJA7HvYv+tNIRHKfUWw/q+v2jdgN4ebz+KY9tGx5J4rHP0o84mNP+ApH66HRX8us3Khqg==", - "requires": { - "foreground-child": "^3.1.0", - "jackspeak": "^2.3.6", - "minimatch": "^9.0.1", - "minipass": "^7.0.4", - "path-scurry": "^1.10.2" - } - }, "graceful-fs": { "version": "4.2.10", "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", @@ -1531,20 +1074,6 @@ "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" }, - "isexe": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/isexe/-/isexe-2.0.0.tgz", - "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==" - }, - "jackspeak": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", - "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", - "requires": { - "@isaacs/cliui": "^8.0.2", - "@pkgjs/parseargs": "^0.11.0" - } - }, "jsonfile": { "version": "6.1.0", "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", @@ -1564,38 +1093,19 @@ "resolved": "https://registry.npmjs.org/long/-/long-5.2.4.tgz", "integrity": "sha512-qtzLbJE8hq7VabR3mISmVGtoXP8KGc2Z/AT8OuqlYD7JTR3oqrgwdjnk07wpj1twXxYmgDXgoKVWUG/fReSzHg==" }, - "lru-cache": { - "version": "10.2.2", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.2.tgz", - "integrity": "sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==" - }, - "minimatch": { - "version": "9.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.4.tgz", - "integrity": "sha512-KqWh+VchfxcMNRAJjj2tnsSJdNbHsVgnkBhTNrW7AjVo6OvLtxw8zfT9oLw1JSohlFzJ8jCoTgaoXvJ+kHt6fw==", - "requires": { - "brace-expansion": "^2.0.1" - } - }, "minipass": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.1.2.tgz", "integrity": "sha512-qOOzS1cBTWYF4BH8fVePDBOO9iptMnGUEZwNc/cMWnTV2nVLZ7VoNWEPHkYczZA0pdoA7dl6e7FL659nX9S2aw==" }, "minizlib": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.0.1.tgz", - "integrity": "sha512-umcy022ILvb5/3Djuu8LWeqUa8D68JaBzlttKeMWen48SjabqS3iY5w/vzeMzMUNhLDifyhbOwKDSznB1vvrwg==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-3.1.0.tgz", + "integrity": "sha512-KZxYo1BUkWD2TVFLr0MQoM8vUUigWD3LlD83a/75BqC+4qE0Hb1Vo5v1FgcfaNXvfXzr+5EhQ6ing/CaBijTlw==", "requires": { - "minipass": "^7.0.4", - "rimraf": "^5.0.5" + "minipass": "^7.1.2" } }, - "mkdirp": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", - "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==" - }, "mkdirp-classic": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", @@ -1607,20 +1117,20 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, "nan": { - "version": "2.22.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.0.tgz", - "integrity": "sha512-nbajikzWTMwsW+eSsNm3QwlOs7het9gGJU5dDZzRTQGk03vyBOauxgI4VakDzE0PtsGTmXPsXTbbjVhRwR5mpw==", + "version": "2.26.2", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.26.2.tgz", + "integrity": "sha512-0tTvBTYkt3tdGw22nrAy50x7gpbGCCFH3AFcyS5WiUu7Eu4vWlri1woE6qHBSfy11vksDqkiwjOnlR7WV8G1Hw==", "optional": true }, "nanoid": { - "version": "5.1.4", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.4.tgz", - "integrity": "sha512-GTFcMIDgR7tqji/LpSY8rtg464VnJl/j6ypoehYnuGb+Y8qZUdtKB8WVCXon0UEZgFDbuUxpIl//6FHLHgXSNA==" + "version": "5.1.7", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-5.1.7.tgz", + "integrity": "sha512-ua3NDgISf6jdwezAheMOk4mbE1LXjm1DfMUDMuJf4AqxLFK3ccGpgWizwa5YV7Yz9EpXwEaWoRXSb/BnV0t5dQ==" }, "nanoid-dictionary": { - "version": "5.0.0-beta.1", - "resolved": "https://registry.npmjs.org/nanoid-dictionary/-/nanoid-dictionary-5.0.0-beta.1.tgz", - "integrity": "sha512-xBkL9zzkNjzJ/UnmWyiOUDVX/COoi05eS0oU28RYKFFQhdnzO5dTOPbVZ/fCFgIOGr1zNinDHJ68mm/KQfcgcw==" + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/nanoid-dictionary/-/nanoid-dictionary-5.0.0.tgz", + "integrity": "sha512-/iCyQHwt36XkaIvSE9fcC8p6DiMPCZMTSMj9UT56Cv6T7f5CuxvOMhpNncaNieQ4z4d32p7ruEtAfRsb7Ya8Gw==" }, "once": { "version": "1.4.0", @@ -1630,20 +1140,6 @@ "wrappy": "1" } }, - "path-key": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", - "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==" - }, - "path-scurry": { - "version": "1.10.2", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.2.tgz", - "integrity": "sha512-7xTavNy5RQXnsjANvVvMkEjvloOinkAjv/Z6Ildz9v2RinZ4SBKTWFOVRbaF8p0vpHnyjV/UwNDdKuUv6M5qcA==", - "requires": { - "lru-cache": "^10.2.0", - "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" - } - }, "protobufjs": { "version": "7.4.0", "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.4.0.tgz", @@ -1664,9 +1160,9 @@ } }, "pump": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", - "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", + "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", "requires": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -1687,14 +1183,6 @@ "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==" }, - "rimraf": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-5.0.5.tgz", - "integrity": "sha512-CqDakW+hMe/Bz202FPEymy68P+G50RfMQK+Qo5YUqc9SPipvbGjCGKd0RSKEelbsfQuw3g5NZDSrlZZAJurH1A==", - "requires": { - "glob": "^10.3.7" - } - }, "safe-buffer": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", @@ -1705,28 +1193,10 @@ "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, - "shebang-command": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", - "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==", - "requires": { - "shebang-regex": "^3.0.0" - } - }, - "shebang-regex": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz", - "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==" - }, - "signal-exit": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", - "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==" - }, "slugify": { - "version": "1.6.6", - "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.6.tgz", - "integrity": "sha512-h+z7HKHYXj6wJU+AnS/+IH8Uh9fdcX1Lrhg1/VMdf9PwoBQXFcXiAdsy2tSK0P6gKwJLXp02r90ahUCqHk9rrw==" + "version": "1.6.9", + "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.9.tgz", + "integrity": "sha512-vZ7rfeehZui7wQs438JXBckYLkIIdfHOXsaVEUMyS5fHo1483l1bMdo0EDSWYclY0yZKFOipDy4KHuKs6ssvdg==" }, "split-ca": { "version": "1.0.1", @@ -1734,14 +1204,14 @@ "integrity": "sha512-Q5thBSxp5t8WPTTJQS59LrGqOZqOsrhDGDVm8azCqIBjSBd7nd9o2PM+mDulQQkh8h//4U6hFZnc/mul8t5pWQ==" }, "ssh2": { - "version": "1.16.0", - "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.16.0.tgz", - "integrity": "sha512-r1X4KsBGedJqo7h8F5c4Ybpcr5RjyP+aWIG007uBPRjmdQWfEiVLzSK71Zji1B9sKxwaCvD8y8cwSkYrlLiRRg==", + "version": "1.17.0", + "resolved": "https://registry.npmjs.org/ssh2/-/ssh2-1.17.0.tgz", + "integrity": "sha512-wPldCk3asibAjQ/kziWQQt1Wh3PgDFpC0XpwclzKcdT1vql6KeYxf5LIt4nlFkUeR8WuphYMKqUA56X4rjbfgQ==", "requires": { "asn1": "^0.2.6", "bcrypt-pbkdf": "^1.0.2", "cpu-features": "~0.0.10", - "nan": "^2.20.0" + "nan": "^2.23.0" } }, "string_decoder": { @@ -1762,16 +1232,6 @@ "strip-ansi": "^6.0.1" } }, - "string-width-cjs": { - "version": "npm:string-width@4.2.3", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", - "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "requires": { - "emoji-regex": "^8.0.0", - "is-fullwidth-code-point": "^3.0.0", - "strip-ansi": "^6.0.1" - } - }, "strip-ansi": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", @@ -1780,24 +1240,15 @@ "ansi-regex": "^5.0.1" } }, - "strip-ansi-cjs": { - "version": "npm:strip-ansi@6.0.1", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", - "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "requires": { - "ansi-regex": "^5.0.1" - } - }, "tar": { - "version": "7.4.3", - "resolved": "https://registry.npmjs.org/tar/-/tar-7.4.3.tgz", - "integrity": "sha512-5S7Va8hKfV7W5U6g3aYxXmlPoZVAwUMy9AOKyF2fVuZa2UD3qZjg578OrLRt8PcNN1PleVaL/5/yYATNL0ICUw==", + "version": "7.5.13", + "resolved": "https://registry.npmjs.org/tar/-/tar-7.5.13.tgz", + "integrity": "sha512-tOG/7GyXpFevhXVh8jOPJrmtRpOTsYqUIkVdVooZYJS/z8WhfQUX8RJILmeuJNinGAMSu1veBr4asSHFt5/hng==", "requires": { "@isaacs/fs-minipass": "^4.0.0", "chownr": "^3.0.0", "minipass": "^7.1.2", - "minizlib": "^3.0.1", - "mkdirp": "^3.0.1", + "minizlib": "^3.1.0", "yallist": "^5.0.0" }, "dependencies": { @@ -1809,14 +1260,14 @@ } }, "tar-fs": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.0.1.tgz", - "integrity": "sha512-6tzWDMeroL87uF/+lin46k+Q+46rAJ0SyPGz7OW7wTgblI273hsBqk2C1j0/xNadNLKDTUL9BukSjB7cwgmlPA==", + "version": "2.1.4", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz", + "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==", "requires": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", "pump": "^3.0.0", - "tar-stream": "^2.0.0" + "tar-stream": "^2.1.4" } }, "tar-stream": { @@ -1829,27 +1280,6 @@ "fs-constants": "^1.0.0", "inherits": "^2.0.3", "readable-stream": "^3.1.1" - }, - "dependencies": { - "bl": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", - "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", - "requires": { - "buffer": "^5.5.0", - "inherits": "^2.0.4", - "readable-stream": "^3.4.0" - } - }, - "buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "requires": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - } } }, "tweetnacl": { @@ -1877,14 +1307,6 @@ "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==" }, - "which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "requires": { - "isexe": "^2.0.0" - } - }, "wrap-ansi": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", @@ -1895,16 +1317,6 @@ "strip-ansi": "^6.0.0" } }, - "wrap-ansi-cjs": { - "version": "npm:wrap-ansi@7.0.0", - "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", - "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "requires": { - "ansi-styles": "^4.0.0", - "string-width": "^4.1.0", - "strip-ansi": "^6.0.0" - } - }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", diff --git a/dev/deploy-to-container/package.json b/dev/deploy-to-container/package.json index eb8cb9012a..ccc78fc63b 100644 --- a/dev/deploy-to-container/package.json +++ b/dev/deploy-to-container/package.json @@ -2,12 +2,12 @@ "name": "deploy-to-container", "type": "module", "dependencies": { - "dockerode": "^4.0.4", - "fs-extra": "^11.3.0", - "nanoid": "5.1.4", - "nanoid-dictionary": "5.0.0-beta.1", - "slugify": "1.6.6", - "tar": "^7.4.3", + "dockerode": "^4.0.10", + "fs-extra": "^11.3.4", + "nanoid": "5.1.7", + "nanoid-dictionary": "5.0.0", + "slugify": "1.6.9", + "tar": "^7.5.13", "yargs": "^17.7.2" }, "engines": { diff --git a/dev/deploy-to-container/settings_local.py b/dev/deploy-to-container/settings_local.py index e878206bd5..055b48d0f5 100644 --- a/dev/deploy-to-container/settings_local.py +++ b/dev/deploy-to-container/settings_local.py @@ -2,8 +2,6 @@ # -*- coding: utf-8 -*- from ietf.settings import * # pyflakes:ignore -from ietf.settings import STORAGES, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS -import botocore.config ALLOWED_HOSTS = ['*'] @@ -73,30 +71,11 @@ DE_GFM_BINARY = '/usr/local/bin/de-gfm' -# No real secrets here, these are public testing values _only_ APP_API_TOKENS = { - "ietf.api.views.ingest_email_test": ["ingestion-test-token"] + "ietf.api.red_api" : ["devtoken", "redtoken"], # Not a real secret + "ietf.api.views.ingest_email_test": ["ingestion-test-token"], # Not a real secret + "ietf.api.views_rpc" : ["devtoken"], # Not a real secret } - # OIDC configuration SITE_URL = 'https://__HOSTNAME__' - -for storagename in MORE_STORAGE_NAMES: - STORAGES[storagename] = { - "BACKEND": "ietf.doc.storage_backends.CustomS3Storage", - "OPTIONS": dict( - endpoint_url="http://blobstore:9000", - access_key="minio_root", - secret_key="minio_pass", - security_token=None, - client_config=botocore.config.Config( - signature_version="s3v4", - connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT, - read_timeout=BLOBSTORAGE_READ_TIMEOUT, - retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS}, - ), - verify=False, - bucket_name=f"test-{storagename}", - ), - } diff --git a/dev/deploy-to-container/start.sh b/dev/deploy-to-container/start.sh index 2c83d6970c..5d976f80ea 100644 --- a/dev/deploy-to-container/start.sh +++ b/dev/deploy-to-container/start.sh @@ -35,6 +35,18 @@ echo "Running Datatracker checks..." # Migrate, adjusting to what the current state of the underlying database might be: +# On production, the blobdb tables are in a separate database. Manipulate migration +# history to ensure that they're created for the sandbox environment that runs it +# all from a single database. +echo "Ensuring blobdb relations exist..." +/usr/local/bin/python ./ietf/manage.py migrate --settings=settings_local --fake blobdb zero +if ! /usr/local/bin/python ./ietf/manage.py migrate --settings=settings_local blobdb; then + # If we are restarting a sandbox, the migration may already have run and re-running + # it will fail. Assume that happened and fake it. + /usr/local/bin/python ./ietf/manage.py migrate --settings=settings_local --fake blobdb +fi + +# Now run the migrations for real echo "Running Datatracker migrations..." /usr/local/bin/python ./ietf/manage.py migrate --settings=settings_local diff --git a/dev/diff/package-lock.json b/dev/diff/package-lock.json index 4e92d2a6e0..d1c2fbd763 100644 --- a/dev/diff/package-lock.json +++ b/dev/diff/package-lock.json @@ -7,7 +7,7 @@ "name": "diff", "dependencies": { "chalk": "^5.4.1", - "dockerode": "^4.0.4", + "dockerode": "^4.0.6", "enquirer": "^2.4.1", "extract-zip": "^2.0.1", "fs-extra": "^11.3.0", @@ -15,7 +15,7 @@ "keypress": "^0.2.1", "listr2": "^6.6.1", "lodash-es": "^4.17.21", - "luxon": "^3.5.0", + "luxon": "^3.6.1", "pretty-bytes": "^6.1.1", "tar": "^7.4.3", "yargs": "^17.7.2" @@ -359,7 +359,8 @@ "type": "consulting", "url": "https://feross.org/support" } - ] + ], + "license": "MIT" }, "node_modules/bcrypt-pbkdf": { "version": "1.0.2", @@ -370,6 +371,17 @@ "tweetnacl": "^0.14.3" } }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, "node_modules/brace-expansion": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", @@ -378,6 +390,30 @@ "balanced-match": "^1.0.0" } }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "node_modules/buffer-crc32": { "version": "0.2.13", "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", @@ -435,7 +471,8 @@ "node_modules/chownr": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", - "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==" + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "license": "ISC" }, "node_modules/cli-cursor": { "version": "4.0.0", @@ -638,9 +675,9 @@ } }, "node_modules/dockerode": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.4.tgz", - "integrity": "sha512-6GYP/EdzEY50HaOxTVTJ2p+mB5xDHTMJhS+UoGrVyS6VC+iQRh7kZ4FRpUYq6nziby7hPqWhOrFFUFTMUZJJ5w==", + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.6.tgz", + "integrity": "sha512-FbVf3Z8fY/kALB9s+P9epCpWhfi/r0N2DgYYcYpsAUlaTxPjdsitsFobnltb+lyCgAIvf9C+4PSWlTnHlJMf1w==", "license": "Apache-2.0", "dependencies": { "@balena/dockerignore": "^1.0.2", @@ -648,7 +685,7 @@ "@grpc/proto-loader": "^0.7.13", "docker-modem": "^5.0.6", "protobufjs": "^7.3.2", - "tar-fs": "~2.0.1", + "tar-fs": "~2.1.2", "uuid": "^10.0.0" }, "engines": { @@ -776,7 +813,8 @@ "node_modules/fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", - "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "license": "MIT" }, "node_modules/fs-extra": { "version": "11.3.0", @@ -895,7 +933,8 @@ "type": "consulting", "url": "https://feross.org/support" } - ] + ], + "license": "BSD-3-Clause" }, "node_modules/inherits": { "version": "2.0.4", @@ -1186,9 +1225,10 @@ } }, "node_modules/luxon": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz", - "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==", + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.6.1.tgz", + "integrity": "sha512-tJLxrKJhO2ukZ5z0gyjY1zPh3Rh88Ej9P7jNrZiHMUXHae1yvI2imgOZtL1TO8TW6biMMKfTtAOoEJANgtWBMQ==", + "license": "MIT", "engines": { "node": ">=12" } @@ -1263,7 +1303,8 @@ "node_modules/mkdirp-classic": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", - "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==" + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "license": "MIT" }, "node_modules/ms": { "version": "2.1.2", @@ -1662,20 +1703,22 @@ } }, "node_modules/tar-fs": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.0.1.tgz", - "integrity": "sha512-6tzWDMeroL87uF/+lin46k+Q+46rAJ0SyPGz7OW7wTgblI273hsBqk2C1j0/xNadNLKDTUL9BukSjB7cwgmlPA==", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.2.tgz", + "integrity": "sha512-EsaAXwxmx8UB7FRKqeozqEPop69DXcmYwTQwXvyAPF352HJsPdkVhvTaDPYqfNgruveJIJy3TA2l+2zj8LJIJA==", + "license": "MIT", "dependencies": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", "pump": "^3.0.0", - "tar-stream": "^2.0.0" + "tar-stream": "^2.1.4" } }, "node_modules/tar-stream": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "license": "MIT", "dependencies": { "bl": "^4.0.3", "end-of-stream": "^1.4.1", @@ -1687,39 +1730,6 @@ "node": ">=6" } }, - "node_modules/tar-stream/node_modules/bl": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", - "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", - "dependencies": { - "buffer": "^5.5.0", - "inherits": "^2.0.4", - "readable-stream": "^3.4.0" - } - }, - "node_modules/tar-stream/node_modules/buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - }, "node_modules/tar/node_modules/chownr": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", @@ -2115,6 +2125,16 @@ "tweetnacl": "^0.14.3" } }, + "bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "requires": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, "brace-expansion": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", @@ -2123,6 +2143,15 @@ "balanced-match": "^1.0.0" } }, + "buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "buffer-crc32": { "version": "0.2.13", "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", @@ -2298,16 +2327,16 @@ } }, "dockerode": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.4.tgz", - "integrity": "sha512-6GYP/EdzEY50HaOxTVTJ2p+mB5xDHTMJhS+UoGrVyS6VC+iQRh7kZ4FRpUYq6nziby7hPqWhOrFFUFTMUZJJ5w==", + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/dockerode/-/dockerode-4.0.6.tgz", + "integrity": "sha512-FbVf3Z8fY/kALB9s+P9epCpWhfi/r0N2DgYYcYpsAUlaTxPjdsitsFobnltb+lyCgAIvf9C+4PSWlTnHlJMf1w==", "requires": { "@balena/dockerignore": "^1.0.2", "@grpc/grpc-js": "^1.11.1", "@grpc/proto-loader": "^0.7.13", "docker-modem": "^5.0.6", "protobufjs": "^7.3.2", - "tar-fs": "~2.0.1", + "tar-fs": "~2.1.2", "uuid": "^10.0.0" } }, @@ -2669,9 +2698,9 @@ "integrity": "sha512-9hp3Vp2/hFQUiIwKo8XCeFVnrg8Pk3TYNPIR7tJADKi5YfcF7vEaK7avFHTlSy3kOKYaJQaalfEo6YuXdceBOQ==" }, "luxon": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.5.0.tgz", - "integrity": "sha512-rh+Zjr6DNfUYR3bPwJEnuwDdqMbxZW7LOQfUN4B54+Cl+0o5zaU9RJ6bcidfDtC1cWCZXQ+nvX8bf6bAji37QQ==" + "version": "3.6.1", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.6.1.tgz", + "integrity": "sha512-tJLxrKJhO2ukZ5z0gyjY1zPh3Rh88Ej9P7jNrZiHMUXHae1yvI2imgOZtL1TO8TW6biMMKfTtAOoEJANgtWBMQ==" }, "mimic-fn": { "version": "2.1.0", @@ -2989,14 +3018,14 @@ } }, "tar-fs": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.0.1.tgz", - "integrity": "sha512-6tzWDMeroL87uF/+lin46k+Q+46rAJ0SyPGz7OW7wTgblI273hsBqk2C1j0/xNadNLKDTUL9BukSjB7cwgmlPA==", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.2.tgz", + "integrity": "sha512-EsaAXwxmx8UB7FRKqeozqEPop69DXcmYwTQwXvyAPF352HJsPdkVhvTaDPYqfNgruveJIJy3TA2l+2zj8LJIJA==", "requires": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", "pump": "^3.0.0", - "tar-stream": "^2.0.0" + "tar-stream": "^2.1.4" } }, "tar-stream": { @@ -3009,27 +3038,6 @@ "fs-constants": "^1.0.0", "inherits": "^2.0.3", "readable-stream": "^3.1.1" - }, - "dependencies": { - "bl": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", - "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", - "requires": { - "buffer": "^5.5.0", - "inherits": "^2.0.4", - "readable-stream": "^3.4.0" - } - }, - "buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "requires": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - } } }, "tweetnacl": { diff --git a/dev/diff/package.json b/dev/diff/package.json index 875e242d3d..a5a7beb0df 100644 --- a/dev/diff/package.json +++ b/dev/diff/package.json @@ -3,7 +3,7 @@ "type": "module", "dependencies": { "chalk": "^5.4.1", - "dockerode": "^4.0.4", + "dockerode": "^4.0.6", "enquirer": "^2.4.1", "extract-zip": "^2.0.1", "fs-extra": "^11.3.0", @@ -11,7 +11,7 @@ "keypress": "^0.2.1", "listr2": "^6.6.1", "lodash-es": "^4.17.21", - "luxon": "^3.5.0", + "luxon": "^3.6.1", "pretty-bytes": "^6.1.1", "tar": "^7.4.3", "yargs": "^17.7.2" diff --git a/dev/diff/settings_local.py b/dev/diff/settings_local.py index 9e0806a8a6..c255cac23d 100644 --- a/dev/diff/settings_local.py +++ b/dev/diff/settings_local.py @@ -2,8 +2,6 @@ # -*- coding: utf-8 -*- from ietf.settings import * # pyflakes:ignore -from ietf.settings import STORAGES, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS -import botocore.config ALLOWED_HOSTS = ['*'] @@ -68,22 +66,3 @@ SLIDE_STAGING_PATH = 'test/staging/' DE_GFM_BINARY = '/usr/local/bin/de-gfm' - -for storagename in MORE_STORAGE_NAMES: - STORAGES[storagename] = { - "BACKEND": "ietf.doc.storage_backends.CustomS3Storage", - "OPTIONS": dict( - endpoint_url="http://blobstore:9000", - access_key="minio_root", - secret_key="minio_pass", - security_token=None, - client_config=botocore.config.Config( - signature_version="s3v4", - connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT, - read_timeout=BLOBSTORAGE_READ_TIMEOUT, - retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS}, - ), - verify=False, - bucket_name=f"test-{storagename}", - ), - } diff --git a/dev/k8s-get-deploy-name/.editorconfig b/dev/k8s-get-deploy-name/.editorconfig new file mode 100644 index 0000000000..fec5c66519 --- /dev/null +++ b/dev/k8s-get-deploy-name/.editorconfig @@ -0,0 +1,7 @@ +[*] +indent_size = 2 +indent_style = space +charset = utf-8 +trim_trailing_whitespace = false +end_of_line = lf +insert_final_newline = true diff --git a/dev/k8s-get-deploy-name/.gitignore b/dev/k8s-get-deploy-name/.gitignore new file mode 100644 index 0000000000..07e6e472cc --- /dev/null +++ b/dev/k8s-get-deploy-name/.gitignore @@ -0,0 +1 @@ +/node_modules diff --git a/dev/k8s-get-deploy-name/.npmrc b/dev/k8s-get-deploy-name/.npmrc new file mode 100644 index 0000000000..580a68c499 --- /dev/null +++ b/dev/k8s-get-deploy-name/.npmrc @@ -0,0 +1,3 @@ +audit = false +fund = false +save-exact = true diff --git a/dev/k8s-get-deploy-name/README.md b/dev/k8s-get-deploy-name/README.md new file mode 100644 index 0000000000..a6605e4dd2 --- /dev/null +++ b/dev/k8s-get-deploy-name/README.md @@ -0,0 +1,16 @@ +# Datatracker Get Deploy Name + +This tool process and slugify a git branch into an appropriate subdomain name. + +## Usage + +1. From the `dev/k8s-get-deploy-name` directory, install the dependencies: +```sh +npm install +``` +2. Run the command: (replacing the `branch` argument) +```sh +node /cli.js --branch feat/fooBar-123 +``` + +The subdomain name will be output. It can then be used in a workflow as a namespace name and subdomain value. diff --git a/dev/k8s-get-deploy-name/cli.js b/dev/k8s-get-deploy-name/cli.js new file mode 100644 index 0000000000..b6c3b5119e --- /dev/null +++ b/dev/k8s-get-deploy-name/cli.js @@ -0,0 +1,22 @@ +#!/usr/bin/env node + +import yargs from 'yargs/yargs' +import { hideBin } from 'yargs/helpers' +import slugify from 'slugify' + +const argv = yargs(hideBin(process.argv)).argv + +let branch = argv.branch +if (!branch) { + throw new Error('Missing --branch argument!') +} +if (branch.indexOf('/') >= 0) { + branch = branch.split('/').slice(1).join('-') +} +branch = slugify(branch, { lower: true, strict: true }) +if (branch.length < 1) { + throw new Error('Branch name is empty!') +} +process.stdout.write(`dt-${branch}`) + +process.exit(0) diff --git a/dev/k8s-get-deploy-name/package-lock.json b/dev/k8s-get-deploy-name/package-lock.json new file mode 100644 index 0000000000..e492a4cd38 --- /dev/null +++ b/dev/k8s-get-deploy-name/package-lock.json @@ -0,0 +1,303 @@ +{ + "name": "k8s-get-deploy-name", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "k8s-get-deploy-name", + "dependencies": { + "slugify": "1.6.6", + "yargs": "17.7.2" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "node_modules/escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "engines": { + "node": ">=6" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "engines": { + "node": ">=8" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/slugify": { + "version": "1.6.6", + "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.6.tgz", + "integrity": "sha512-h+z7HKHYXj6wJU+AnS/+IH8Uh9fdcX1Lrhg1/VMdf9PwoBQXFcXiAdsy2tSK0P6gKwJLXp02r90ahUCqHk9rrw==", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "engines": { + "node": ">=12" + } + } + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" + }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "requires": { + "color-convert": "^2.0.1" + } + }, + "cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "requires": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" + }, + "get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" + }, + "require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==" + }, + "slugify": { + "version": "1.6.6", + "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.6.tgz", + "integrity": "sha512-h+z7HKHYXj6wJU+AnS/+IH8Uh9fdcX1Lrhg1/VMdf9PwoBQXFcXiAdsy2tSK0P6gKwJLXp02r90ahUCqHk9rrw==" + }, + "string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + } + }, + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "requires": { + "ansi-regex": "^5.0.1" + } + }, + "wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "requires": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + } + }, + "y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==" + }, + "yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "requires": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + } + }, + "yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==" + } + } +} diff --git a/dev/k8s-get-deploy-name/package.json b/dev/k8s-get-deploy-name/package.json new file mode 100644 index 0000000000..849f5d9b8d --- /dev/null +++ b/dev/k8s-get-deploy-name/package.json @@ -0,0 +1,8 @@ +{ + "name": "k8s-get-deploy-name", + "type": "module", + "dependencies": { + "slugify": "1.6.6", + "yargs": "17.7.2" + } +} diff --git a/dev/tests/settings_local.py b/dev/tests/settings_local.py index f2166053a7..e1ffd60edb 100644 --- a/dev/tests/settings_local.py +++ b/dev/tests/settings_local.py @@ -2,8 +2,6 @@ # -*- coding: utf-8 -*- from ietf.settings import * # pyflakes:ignore -from ietf.settings import STORAGES, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS -import botocore.config ALLOWED_HOSTS = ['*'] @@ -67,22 +65,3 @@ SLIDE_STAGING_PATH = 'test/staging/' DE_GFM_BINARY = '/usr/local/bin/de-gfm' - -for storagename in MORE_STORAGE_NAMES: - STORAGES[storagename] = { - "BACKEND": "ietf.doc.storage_backends.CustomS3Storage", - "OPTIONS": dict( - endpoint_url="http://blobstore:9000", - access_key="minio_root", - secret_key="minio_pass", - security_token=None, - client_config=botocore.config.Config( - signature_version="s3v4", - connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT, - read_timeout=BLOBSTORAGE_READ_TIMEOUT, - retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS}, - ), - verify=False, - bucket_name=f"test-{storagename}", - ), - } diff --git a/docker-compose.yml b/docker-compose.yml index 9910c02a99..073d04b896 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,9 +13,10 @@ services: # network_mode: service:db depends_on: + - blobdb + - blobstore - db - mq - - blobstore ipc: host @@ -36,7 +37,7 @@ services: db: image: ghcr.io/ietf-tools/datatracker-db:latest # build: - # context: .. + # context: . # dockerfile: docker/db.Dockerfile restart: unless-stopped volumes: @@ -79,7 +80,36 @@ services: command: - '--loglevel=INFO' depends_on: + - blobdb + - blobstore + - db + - mq + restart: unless-stopped + stop_grace_period: 1m + volumes: + - .:/workspace + - app-assets:/assets + + replicator: + build: + context: . + dockerfile: docker/celery.Dockerfile + init: true + environment: + CELERY_APP: ietf + CELERY_ROLE: worker + UPDATE_REQUIREMENTS_FROM: requirements.txt + DEV_MODE: "yes" + command: + - '--loglevel=INFO' + - '--queues=blobdb' + - '--concurrency=1' + + depends_on: + - blobdb + - blobstore - db + - mq restart: unless-stopped stop_grace_period: 1m volumes: @@ -92,13 +122,33 @@ services: volumes: - "minio-data:/data" + blobdb: + image: postgres:17 + restart: unless-stopped + environment: + POSTGRES_DB: blob + POSTGRES_USER: dt + POSTGRES_PASSWORD: abcd1234 + volumes: + - blobdb-data:/var/lib/postgresql/data +# typesense: +# image: typesense/typesense:30.1 +# restart: on-failure +# ports: +# - "8108:8108" +# volumes: +# - ./typesense-data:/data +# command: +# - '--data-dir=/data' +# - '--api-key=typesense-api-key' +# - '--enable-cors' # Celery Beat is a periodic task runner. It is not normally needed for development, # but can be enabled by uncommenting the following. # # beat: -# image: ghcr.io/ietf-tools/datatracker-celery:latest +# image: "${COMPOSE_PROJECT_NAME}-celery" # init: true # environment: # CELERY_APP: ietf @@ -118,3 +168,4 @@ volumes: postgresdb-data: app-assets: minio-data: + blobdb-data: diff --git a/docker/README.md b/docker/README.md index f2161a173f..0ca79a6e89 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,5 +1,18 @@ # Datatracker Development in Docker +- [Getting started](#getting-started) +- [Using Visual Studio Code](#using-visual-studio-code) + - [Initial Setup](#initial-setup) + - [Subsequent Launch](#subsequent-launch) + - [Usage](#usage) +- [Using Other Editors / Generic](#using-other-editors--generic) + - [Exit Environment](#exit-environment) + - [Accessing PostgreSQL Port](#accessing-postgresql-port) +- [Clean and Rebuild DB from latest image](#clean-and-rebuild-db-from-latest-image) +- [Clean all](#clean-all) +- [Updating an older environment](#updating-an-older-environment) +- [Notes / Troubleshooting](#notes--troubleshooting) + ## Getting started 1. [Set up Docker](https://docs.docker.com/get-started/) on your preferred platform. On Windows, it is highly recommended to use the [WSL 2 *(Windows Subsystem for Linux)*](https://docs.docker.com/desktop/windows/wsl/) backend. @@ -123,7 +136,14 @@ docker compose down to terminate the containers. -### Clean and Rebuild DB from latest image +### Accessing PostgreSQL Port + +The port is exposed but not automatically mapped to `5432` to avoid potential conflicts with the host. To get the mapped port, run the command *(from the project `/docker` directory)*: +```sh +docker compose port db 5432 +``` + +## Clean and Rebuild DB from latest image To delete the active DB container, its volume and get the latest image / DB dump, simply run the following command: @@ -141,7 +161,7 @@ docker compose pull db docker compose build --no-cache db ``` -### Clean all +## Clean all To delete all containers for this project, its associated images and purge any remaining dangling images, simply run the following command: @@ -157,17 +177,20 @@ On Windows: docker compose down -v --rmi all docker image prune ``` -### Updating an older environment + +## Updating an older environment If you already have a clone, such as from a previous codesprint, and are updating that clone, before starting the datatracker from the updated image: -* rm ietf/settings_local.py # The startup script will put a new one, appropriate to the current release, in place -* Execute the `Clean all` sequence above. +1. `rm ietf/settings_local.py` *(The startup script will put a new one, appropriate to the current release, in place)* +1. Execute the [Clean all](#clean-all) sequence above. -### Accessing PostgreSQL Port +If the dev environment fails to start, even after running the [Clean all](#clean-all) sequence above, you can fully purge all docker cache, containers, images and volumes by running the command below. + +> [!CAUTION] +> Note that this will delete everything docker-related, including non-datatracker docker resources you might have. -The port is exposed but not automatically mapped to `5432` to avoid potential conflicts with the host. To get the mapped port, run the command *(from the project `/docker` directory)*: ```sh -docker compose port db 5432 +docker system prune -a --volumes ``` ## Notes / Troubleshooting diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile index fee3833733..dd4cf72ffd 100644 --- a/docker/app.Dockerfile +++ b/docker/app.Dockerfile @@ -10,12 +10,7 @@ ARG USER_GID=$USER_UID COPY docker/scripts/app-setup-debian.sh /tmp/library-scripts/docker-setup-debian.sh RUN sed -i 's/\r$//' /tmp/library-scripts/docker-setup-debian.sh && chmod +x /tmp/library-scripts/docker-setup-debian.sh -# Add Postgresql Apt Repository to get 14 -RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list -RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - - RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ - && apt-get install -y --no-install-recommends postgresql-client-14 pgloader \ # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131 && apt-get purge -y imagemagick imagemagick-6-common \ # Install common packages, non-root user diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile index f364456c7a..2501636049 100644 --- a/docker/base.Dockerfile +++ b/docker/base.Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-bookworm +FROM python:3.12-bookworm LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive @@ -11,21 +11,22 @@ RUN apt-get update \ # Add Node.js Source RUN apt-get install -y --no-install-recommends ca-certificates curl gnupg \ - && mkdir -p /etc/apt/keyrings\ - && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg -RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list -RUN echo "Package: nodejs" >> /etc/apt/preferences.d/preferences && \ - echo "Pin: origin deb.nodesource.com" >> /etc/apt/preferences.d/preferences && \ - echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/preferences + && mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \ + && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list +RUN echo "Package: nodejs" >> /etc/apt/preferences.d/preferences \ + && echo "Pin: origin deb.nodesource.com" >> /etc/apt/preferences.d/preferences \ + && echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/preferences # Add Docker Source -RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg -RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \ - $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null +RUN mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | tee /etc/apt/sources.list.d/docker.list -# Add PostgreSQL Source -RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list -RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - +# Add PostgreSQL Source +RUN mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /etc/apt/keyrings/apt.postgresql.org.gpg \ + && echo "deb [signed-by=/etc/apt/keyrings/apt.postgresql.org.gpg] https://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list # Install the packages we need RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends \ @@ -69,7 +70,7 @@ RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends nodejs \ pgloader \ pigz \ - postgresql-client-14 \ + postgresql-client-17 \ pv \ python3-ipython \ ripgrep \ diff --git a/docker/celery.Dockerfile b/docker/celery.Dockerfile index e44200398c..e93ca3cf77 100644 --- a/docker/celery.Dockerfile +++ b/docker/celery.Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-celery:latest +FROM ghcr.io/ietf-tools/datatracker-app-base:latest LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive @@ -10,12 +10,7 @@ ARG USER_GID=$USER_UID COPY docker/scripts/app-setup-debian.sh /tmp/library-scripts/docker-setup-debian.sh RUN sed -i 's/\r$//' /tmp/library-scripts/docker-setup-debian.sh && chmod +x /tmp/library-scripts/docker-setup-debian.sh -# Add Postgresql Apt Repository to get 14 -RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list -RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - - RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ - && apt-get install -y --no-install-recommends postgresql-client-14 pgloader \ # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131 && apt-get purge -y imagemagick imagemagick-6-common \ # Install common packages, non-root user @@ -34,7 +29,7 @@ RUN bash /tmp/library-scripts/docker-setup-python.sh "none" "/usr/local" "${PIPX RUN rm -rf /tmp/library-scripts # Copy the startup file -COPY dev/celery/docker-init.sh /docker-init.sh +COPY docker/scripts/app-init-celery.sh /docker-init.sh RUN sed -i 's/\r$//' /docker-init.sh && \ chmod +x /docker-init.sh diff --git a/docker/configs/nginx-proxy.conf b/docker/configs/nginx-proxy.conf index 3068cc71d7..e1f739970b 100644 --- a/docker/configs/nginx-proxy.conf +++ b/docker/configs/nginx-proxy.conf @@ -1,9 +1,14 @@ +upstream datatracker_backend { + server 127.0.0.1:8001; +} + server { listen 8000 default_server; listen [::]:8000 default_server; proxy_read_timeout 1d; proxy_send_timeout 1d; + client_max_body_size 0; # disable checking root /var/www/html; index index.html index.htm index.nginx-debian.html; @@ -23,7 +28,7 @@ server { location / { error_page 502 /502.html; - proxy_pass http://localhost:8001/; + proxy_pass http://datatracker_backend/; proxy_set_header Host localhost:8000; } diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index 46833451c1..94adc516a4 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -2,12 +2,26 @@ # -*- coding: utf-8 -*- from ietf.settings import * # pyflakes:ignore -from ietf.settings import STORAGES, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS -import botocore.config +from ietf.settings import ( + ARTIFACT_STORAGE_NAMES, + STORAGES, + BLOBSTORAGE_MAX_ATTEMPTS, + BLOBSTORAGE_READ_TIMEOUT, + BLOBSTORAGE_CONNECT_TIMEOUT, +) ALLOWED_HOSTS = ['*'] from ietf.settings_postgresqldb import DATABASES # pyflakes:ignore +DATABASE_ROUTERS = ["ietf.blobdb.routers.BlobdbStorageRouter"] +BLOBDB_DATABASE = "blobdb" +BLOBDB_REPLICATION = { + "ENABLED": True, + "DEST_STORAGE_PATTERN": "r2-{bucket}", + "INCLUDE_BUCKETS": ARTIFACT_STORAGE_NAMES, + "EXCLUDE_BUCKETS": ["staging"], + "VERBOSE_LOGGING": True, +} IDSUBMIT_IDNITS_BINARY = "/usr/local/bin/idnits" IDSUBMIT_STAGING_PATH = "/assets/www6s/staging/" @@ -39,25 +53,6 @@ # DEV_TEMPLATE_CONTEXT_PROCESSORS = [ # 'ietf.context_processors.sql_debug', # ] -for storagename in MORE_STORAGE_NAMES: - STORAGES[storagename] = { - "BACKEND": "ietf.doc.storage_backends.CustomS3Storage", - "OPTIONS": dict( - endpoint_url="http://blobstore:9000", - access_key="minio_root", - secret_key="minio_pass", - security_token=None, - client_config=botocore.config.Config( - signature_version="s3v4", - connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT, - read_timeout=BLOBSTORAGE_READ_TIMEOUT, - retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS}, - ), - verify=False, - bucket_name=storagename, - ), - } - DOCUMENT_PATH_PATTERN = '/assets/ietfdata/doc/{doc.type_id}/' INTERNET_DRAFT_PATH = '/assets/ietf-ftp/internet-drafts/' @@ -80,3 +75,48 @@ STATIC_IETF_ORG = "/_static" STATIC_IETF_ORG_INTERNAL = "http://static" + + +# Blob replication storage for dev +import botocore.config +for storagename in ARTIFACT_STORAGE_NAMES: + replica_storagename = f"r2-{storagename}" + STORAGES[replica_storagename] = { + "BACKEND": "ietf.doc.storage.MetadataS3Storage", + "OPTIONS": dict( + endpoint_url="http://blobstore:9000", + access_key="minio_root", + secret_key="minio_pass", + security_token=None, + client_config=botocore.config.Config( + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + signature_version="s3v4", + connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT, + read_timeout=BLOBSTORAGE_READ_TIMEOUT, + retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS}, + ), + verify=False, + bucket_name=f"{storagename}", + ), + } + +# For dev on rfc-index generation, create a red_bucket/ directory in the project root +# and uncomment these settings. Generated files will appear in this directory. To +# generate an accurate index, put up-to-date copies of unusable-rfc-numbers.json, +# april-first-rfc-numbers.json, and publication-std-levels.json in this directory +# before generating the index. +# +# STORAGES["red_bucket"] = { +# "BACKEND": "django.core.files.storage.FileSystemStorage", +# "OPTIONS": {"location": "red_bucket"}, +# } + +APP_API_TOKENS = { + "ietf.api.red_api" : ["devtoken", "redtoken"], # Not a real secret + "ietf.api.views_rpc" : ["devtoken"], # Not a real secret +} + +# Errata system api configuration +ERRATA_METADATA_NOTIFICATION_URL = "http://host.docker.internal:8808/api/rfc_metadata_update/" +ERRATA_METADATA_NOTIFICATION_API_KEY = "not a real secret" diff --git a/docker/configs/settings_local_vite.py b/docker/configs/settings_local_vite.py index 7fb12a003d..9116905b12 100644 --- a/docker/configs/settings_local_vite.py +++ b/docker/configs/settings_local_vite.py @@ -2,5 +2,9 @@ # -*- coding: utf-8 -*- from ietf.settings_local import * # pyflakes:ignore +from ietf.settings_local import DJANGO_VITE -DJANGO_VITE_DEV_MODE = True +DJANGO_VITE["default"] |= { + "dev_mode": True, + "dev_server_port": 3000, +} diff --git a/docker/configs/settings_postgresqldb.py b/docker/configs/settings_postgresqldb.py index 05d19b9a86..9b98586658 100644 --- a/docker/configs/settings_postgresqldb.py +++ b/docker/configs/settings_postgresqldb.py @@ -7,4 +7,12 @@ 'USER': 'django', 'PASSWORD': 'RkTkDPFnKpko', }, + 'blobdb': { + 'HOST': 'blobdb', + 'PORT': 5432, + 'NAME': 'blob', + 'ENGINE': 'django.db.backends.postgresql', + 'USER': 'dt', + 'PASSWORD': 'abcd1234', + }, } diff --git a/docker/db.Dockerfile b/docker/db.Dockerfile index 31c41011e6..48ab298780 100644 --- a/docker/db.Dockerfile +++ b/docker/db.Dockerfile @@ -1,7 +1,7 @@ # ===================== # --- Builder Stage --- # ===================== -FROM postgres:16 AS builder +FROM postgres:17 AS builder ENV POSTGRES_PASSWORD=hk2j22sfiv ENV POSTGRES_USER=django @@ -19,7 +19,7 @@ RUN ["/usr/local/bin/docker-entrypoint.sh", "postgres"] # =================== # --- Final Image --- # =================== -FROM postgres:16 +FROM postgres:17 LABEL maintainer="IETF Tools Team " COPY --from=builder /data $PGDATA @@ -28,3 +28,10 @@ ENV POSTGRES_PASSWORD=hk2j22sfiv ENV POSTGRES_USER=django ENV POSTGRES_DB=datatracker ENV POSTGRES_HOST_AUTH_METHOD=trust + +# build-args for db dump tagging - exposed in the environment and +# in image metadata +ARG datatracker_dumpinfo_date="" +ENV DATATRACKER_DUMPINFO_DATE=$datatracker_dumpinfo_date +ARG datatracker_snapshot="" +ENV DATATRACKER_SNAPSHOT=$datatracker_snapshot diff --git a/docker/docker-compose.celery.yml b/docker/docker-compose.celery.yml deleted file mode 100644 index b6cc3d09e8..0000000000 --- a/docker/docker-compose.celery.yml +++ /dev/null @@ -1,47 +0,0 @@ -services: - mq: - image: rabbitmq:3-alpine - user: '${RABBITMQ_UID:-499:499}' - hostname: datatracker-mq -# deploy: -# resources: -# limits: -# memory: 1gb # coordinate with settings in rabbitmq.conf -# reservations: -# memory: 512mb - mem_limit: 1gb # coordinate with settings in rabbitmq.conf - ports: - - '${MQ_PORT:-5672}:5672' - volumes: - - ./lib.rabbitmq:/var/lib/rabbitmq - - ./rabbitmq.conf:/etc/rabbitmq/conf.d/90-ietf.conf - - ./definitions.json:/ietf-conf/definitions.json - restart: unless-stopped - logging: - driver: "syslog" - options: - syslog-address: 'unixgram:///dev/log' - tag: 'docker/{{.Name}}' -# syslog-address: "tcp://ietfa.amsl.com:514" - - celery: - image: ghcr.io/ietf-tools/datatracker-celery:latest - environment: - CELERY_APP: ietf - # UPDATE_REQUIREMENTS: 1 # uncomment to update Python requirements on startup - command: - - '--loglevel=INFO' - user: '${CELERY_UID:-499:499}' - volumes: - - '${DATATRACKER_PATH:-..}:/workspace' - - '${MYSQL_SOCKET_PATH:-/run/mysql}:/run/mysql' - depends_on: - - mq - network_mode: 'service:mq' - restart: unless-stopped - logging: - driver: "syslog" - options: - syslog-address: 'unixgram:///dev/log' - tag: 'docker/{{.Name}}' -# syslog-address: "tcp://ietfa.amsl.com:514" diff --git a/docker/docker-compose.extend.yml b/docker/docker-compose.extend.yml index a69a453110..12ebe447d5 100644 --- a/docker/docker-compose.extend.yml +++ b/docker/docker-compose.extend.yml @@ -18,8 +18,8 @@ services: - '5433' blobstore: ports: - - '9000' - - '9001' + - '9000:9000' + - '9001:9001' celery: volumes: - .:/workspace diff --git a/docker/scripts/app-configure-blobstore.py b/docker/scripts/app-configure-blobstore.py index ae87bf1afe..9ae64e0041 100755 --- a/docker/scripts/app-configure-blobstore.py +++ b/docker/scripts/app-configure-blobstore.py @@ -7,7 +7,7 @@ import os import sys -from ietf.settings import MORE_STORAGE_NAMES +from ietf.settings import ARTIFACT_STORAGE_NAMES def init_blobstore(): @@ -17,13 +17,20 @@ def init_blobstore(): aws_access_key_id=os.environ.get("BLOB_STORE_ACCESS_KEY", "minio_root"), aws_secret_access_key=os.environ.get("BLOB_STORE_SECRET_KEY", "minio_pass"), aws_session_token=None, - config=botocore.config.Config(signature_version="s3v4"), + config=botocore.config.Config( + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + signature_version="s3v4", + ), ) - for bucketname in MORE_STORAGE_NAMES: + for bucketname in ARTIFACT_STORAGE_NAMES: + adjusted_bucket_name = ( + os.environ.get("BLOB_STORE_BUCKET_PREFIX", "") + + bucketname + + os.environ.get("BLOB_STORE_BUCKET_SUFFIX", "") + ).strip() try: - blobstore.create_bucket( - Bucket=f"{os.environ.get('BLOB_STORE_BUCKET_PREFIX', '')}{bucketname}".strip() - ) + blobstore.create_bucket(Bucket=adjusted_bucket_name) except botocore.exceptions.ClientError as err: if err.response["Error"]["Code"] == "BucketAlreadyExists": print(f"Bucket {bucketname} already exists") @@ -32,5 +39,6 @@ def init_blobstore(): else: print(f"Bucket {bucketname} created") + if __name__ == "__main__": sys.exit(init_blobstore()) diff --git a/dev/celery/docker-init.sh b/docker/scripts/app-init-celery.sh similarity index 85% rename from dev/celery/docker-init.sh rename to docker/scripts/app-init-celery.sh index 9940dfd7d0..17925633d2 100755 --- a/dev/celery/docker-init.sh +++ b/docker/scripts/app-init-celery.sh @@ -90,19 +90,29 @@ if [[ "${CELERY_ROLE}" == "worker" ]]; then run_as_celery_uid /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py check fi +USER_BIN_PATH="/home/dev/.local/bin" +WATCHMEDO="$USER_BIN_PATH/watchmedo" +# Find a celery that works +if [[ -x "$USER_BIN_PATH/celery" ]]; then + # This branch is used for dev + CELERY="$USER_BIN_PATH/celery" +else + # This branch is used for sandbox instances + CELERY="/usr/local/bin/celery" +fi trap 'trap "" TERM; cleanup' TERM # start celery in the background so we can trap the TERM signal -if [[ -n "${DEV_MODE}" ]]; then - watchmedo auto-restart \ +if [[ -n "${DEV_MODE}" && -x "${WATCHMEDO}" ]]; then + $WATCHMEDO auto-restart \ --patterns '*.py' \ --directory 'ietf' \ --recursive \ --debounce-interval 5 \ -- \ - celery --app="${CELERY_APP:-ietf}" "${CELERY_OPTS[@]}" "$@" & + $CELERY --app="${CELERY_APP:-ietf}" "${CELERY_OPTS[@]}" $@ & celery_pid=$! else - celery --app="${CELERY_APP:-ietf}" "${CELERY_OPTS[@]}" "$@" & + $CELERY --app="${CELERY_APP:-ietf}" "${CELERY_OPTS[@]}" "$@" & celery_pid=$! fi diff --git a/docker/scripts/app-init.sh b/docker/scripts/app-init.sh index e970398ac2..1d895cdf53 100755 --- a/docker/scripts/app-init.sh +++ b/docker/scripts/app-init.sh @@ -101,12 +101,14 @@ echo "Running initial checks..." /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py check --settings=settings_local # Migrate, adjusting to what the current state of the underlying database might be: - /usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --fake-initial --settings=settings_local +# Apply migrations to the blobdb database as well (most are skipped) +/usr/local/bin/python $WORKSPACEDIR/ietf/manage.py migrate --settings=settings_local --database=blobdb + if [ -z "$EDITOR_VSCODE" ]; then CODE=0 - python -m smtpd -n -c DebuggingServer localhost:2025 & + python -m aiosmtpd -n -c ietf.utils.aiosmtpd.DevDebuggingHandler -l localhost:2025 & if [ -z "$*" ]; then echo "-----------------------------------------------------------------" echo "Ready!" diff --git a/ietf/api/__init__.py b/ietf/api/__init__.py index 9fadab8e6f..d4562f97dd 100644 --- a/ietf/api/__init__.py +++ b/ietf/api/__init__.py @@ -4,6 +4,7 @@ import datetime import re +import sys from urllib.parse import urlencode @@ -25,6 +26,9 @@ OMITTED_APPS_APIS = ["ietf.status"] +# Pre-py3.11, fromisoformat() does not handle Z or +HH tz offsets +HAVE_BROKEN_FROMISOFORMAT = sys.version_info < (3, 11, 0, "", 0) + def populate_api_list(): _module_dict = globals() for app_config in django_apps.get_app_configs(): @@ -58,6 +62,35 @@ def generate_cache_key(self, *args, **kwargs): # Use a list plus a ``.join()`` because it's faster than concatenation. return "%s:%s:%s:%s" % (self._meta.api_name, self._meta.resource_name, ':'.join(args), smooshed) + def _z_aware_fromisoformat(self, value: str) -> datetime.datetime: + """datetime.datetime.fromisoformat replacement that works with python < 3.11""" + if HAVE_BROKEN_FROMISOFORMAT: + if value.upper().endswith("Z"): + value = value[:-1] + "+00:00" # Z -> UTC + elif re.match(r"[+-][0-9][0-9]$", value[-3:]): + value = value + ":00" # -04 -> -04:00 + return datetime.datetime.fromisoformat(value) + + def filter_value_to_python( + self, value, field_name, filters, filter_expr, filter_type + ): + py_value = super().filter_value_to_python( + value, field_name, filters, filter_expr, filter_type + ) + if isinstance( + self.fields[field_name], tastypie.fields.DateTimeField + ) and isinstance(py_value, str): + # Ensure datetime values are TZ-aware, using UTC by default + try: + dt = self._z_aware_fromisoformat(py_value) + except ValueError: + pass # let tastypie deal with the original value + else: + if dt.tzinfo is None: + dt = dt.replace(tzinfo=datetime.timezone.utc) + py_value = dt.isoformat() + return py_value + TIMEDELTA_REGEX = re.compile(r'^(?P\d+d)?\s?(?P\d+h)?\s?(?P\d+m)?\s?(?P\d+s?)$') @@ -145,5 +178,26 @@ def dehydrate(self, bundle, for_list=True): class Serializer(tastypie.serializers.Serializer): + OPTION_ESCAPE_NULLS = "datatracker-escape-nulls" + def format_datetime(self, data): - return data.astimezone(datetime.timezone.utc).replace(tzinfo=None).isoformat(timespec="seconds") + "Z" + return data.astimezone(datetime.UTC).replace(tzinfo=None).isoformat(timespec="seconds") + "Z" + + def to_simple(self, data, options): + options = options or {} + simple_data = super().to_simple(data, options) + if ( + options.get(self.OPTION_ESCAPE_NULLS, False) + and isinstance(simple_data, str) + ): + # replace nulls with unicode "symbol for null character", \u2400 + simple_data = simple_data.replace("\x00", "\u2400") + return simple_data + + def to_etree(self, data, options=None, name=None, depth=0): + # lxml does not escape nulls on its own, so ask to_simple() to do it. + # This is mostly (only?) an issue when generating errors responses for + # fuzzers. + options = options or {} + options[self.OPTION_ESCAPE_NULLS] = True + return super().to_etree(data, options, name, depth) diff --git a/ietf/api/routers.py b/ietf/api/routers.py index 745ddaa811..99afdb242a 100644 --- a/ietf/api/routers.py +++ b/ietf/api/routers.py @@ -3,14 +3,29 @@ from django.core.exceptions import ImproperlyConfigured from rest_framework import routers -class PrefixedSimpleRouter(routers.SimpleRouter): - """SimpleRouter that adds a dot-separated prefix to its basename""" + +class PrefixedBasenameMixin: + """Mixin to add a prefix to the basename of a rest_framework BaseRouter""" def __init__(self, name_prefix="", *args, **kwargs): self.name_prefix = name_prefix if len(self.name_prefix) == 0 or self.name_prefix[-1] == ".": raise ImproperlyConfigured("Cannot use a name_prefix that is empty or ends with '.'") super().__init__(*args, **kwargs) - def get_default_basename(self, viewset): - basename = super().get_default_basename(viewset) - return f"{self.name_prefix}.{basename}" + def register(self, prefix, viewset, basename=None): + # Get the superclass "register" method from the class this is mixed-in with. + # This avoids typing issues with calling super().register() directly in a + # mixin class. + super_register = getattr(super(), "register") + if not super_register or not callable(super_register): + raise TypeError("Must mixin with superclass that has register() method") + super_register(prefix, viewset, basename=f"{self.name_prefix}.{basename}") + + +class PrefixedSimpleRouter(PrefixedBasenameMixin, routers.SimpleRouter): + """SimpleRouter that adds a dot-separated prefix to its basename""" + + +class PrefixedDefaultRouter(PrefixedBasenameMixin, routers.DefaultRouter): + """DefaultRouter that adds a dot-separated prefix to its basename""" + diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py new file mode 100644 index 0000000000..0e45ee3b39 --- /dev/null +++ b/ietf/api/serializers_rpc.py @@ -0,0 +1,871 @@ +# Copyright The IETF Trust 2025-2026, All Rights Reserved +import datetime +from pathlib import Path +from typing import Literal, Optional + +from django.db import transaction +from django.urls import reverse as urlreverse +from django.utils import timezone +from drf_spectacular.types import OpenApiTypes +from drf_spectacular.utils import extend_schema_field +from rest_framework import fields, serializers + +from ietf.doc.expire import move_draft_files_to_archive +from ietf.doc.models import ( + DocumentAuthor, + Document, + RelatedDocument, + State, + DocEvent, + RfcAuthor, +) +from ietf.doc.serializers import RfcAuthorSerializer +from ietf.doc.tasks import trigger_red_precomputer_task, update_rfc_searchindex_task +from ietf.doc.utils import ( + default_consensus, + prettify_std_name, + update_action_holders, + update_rfcauthors, +) +from ietf.group.models import Group, Role +from ietf.group.serializers import AreaSerializer +from ietf.name.models import StreamName, StdLevelName +from ietf.person.models import Person +from ietf.utils import log + + +class PersonSerializer(serializers.ModelSerializer): + email = serializers.EmailField(read_only=True) + picture = serializers.URLField(source="cdn_photo_url", read_only=True) + url = serializers.SerializerMethodField( + help_text="relative URL for datatracker person page" + ) + + class Meta: + model = Person + fields = ["id", "plain_name", "email", "picture", "url"] + read_only_fields = ["id", "plain_name", "email", "picture", "url"] + + @extend_schema_field(OpenApiTypes.URI) + def get_url(self, object: Person): + return urlreverse( + "ietf.person.views.profile", + kwargs={"email_or_name": object.email_address() or object.name}, + ) + + +class EmailPersonSerializer(serializers.Serializer): + email = serializers.EmailField(source="address") + person_pk = serializers.IntegerField(source="person.pk") + name = serializers.CharField(source="person.name") + last_name = serializers.CharField(source="person.last_name") + initials = serializers.CharField(source="person.initials") + + +class LowerCaseEmailField(serializers.EmailField): + def to_representation(self, value): + return super().to_representation(value).lower() + + +class AuthorPersonSerializer(serializers.ModelSerializer): + person_pk = serializers.IntegerField(source="pk", read_only=True) + last_name = serializers.CharField() + initials = serializers.CharField() + email_addresses = serializers.ListField( + source="email_set.all", child=LowerCaseEmailField() + ) + + class Meta: + model = Person + fields = ["person_pk", "name", "last_name", "initials", "email_addresses"] + + +class RfcWithAuthorsSerializer(serializers.ModelSerializer): + authors = AuthorPersonSerializer(many=True, source="author_persons") + + class Meta: + model = Document + fields = ["rfc_number", "authors"] + + +class DraftWithAuthorsSerializer(serializers.ModelSerializer): + draft_name = serializers.CharField(source="name") + authors = AuthorPersonSerializer(many=True, source="author_persons") + + class Meta: + model = Document + fields = ["draft_name", "authors"] + + +class WgChairSerializer(serializers.Serializer): + """Serialize a WG chair's name and email from a Role""" + + name = serializers.SerializerMethodField() + email = serializers.SerializerMethodField() + + @extend_schema_field(serializers.CharField) + def get_name(self, role: Role) -> str: + return role.person.plain_name() + + @extend_schema_field(serializers.EmailField) + def get_email(self, role: Role) -> str: + return role.email.email_address() + + +class DocumentAuthorSerializer(serializers.ModelSerializer): + """Serializer for a Person in a response""" + + plain_name = serializers.SerializerMethodField() + + class Meta: + model = DocumentAuthor + fields = ["person", "plain_name", "affiliation"] + + def get_plain_name(self, document_author: DocumentAuthor) -> str: + return document_author.person.plain_name() + + +class FullDraftSerializer(serializers.ModelSerializer): + # Redefine these fields so they don't pick up the regex validator patterns. + # There seem to be some non-compliant drafts in the system! If this serializer + # is used for a writeable view, the validation will need to be added back. + name = serializers.CharField(max_length=255) + title = serializers.CharField(max_length=255) + group = serializers.SlugRelatedField(slug_field="acronym", read_only=True) + area = AreaSerializer(read_only=True) + + # Other fields we need to add / adjust + source_format = serializers.SerializerMethodField() + authors = DocumentAuthorSerializer(many=True, source="documentauthor_set") + shepherd = serializers.PrimaryKeyRelatedField( + source="shepherd.person", read_only=True + ) + consensus = serializers.SerializerMethodField() + wg_chairs = serializers.SerializerMethodField() + + class Meta: + model = Document + fields = [ + "id", + "name", + "rev", + "stream", + "title", + "group", + "area", + "abstract", + "pages", + "source_format", + "authors", + "intended_std_level", + "consensus", + "shepherd", + "ad", + "wg_chairs", + ] + + def get_consensus(self, doc: Document) -> Optional[bool]: + return default_consensus(doc) + + @extend_schema_field(WgChairSerializer(many=True)) + def get_wg_chairs(self, doc: Document): + if doc.group is None: + return [] + chairs = doc.group.role_set.filter(name_id="chair").select_related( + "person", "email" + ) + return WgChairSerializer(chairs, many=True).data + + def get_source_format( + self, doc: Document + ) -> Literal["unknown", "xml-v2", "xml-v3", "txt"]: + submission = doc.submission() + if submission is None: + return "unknown" + if ".xml" in submission.file_types: + if submission.xml_version == "3": + return "xml-v3" + else: + return "xml-v2" + elif ".txt" in submission.file_types: + return "txt" + return "unknown" + + +class DraftSerializer(FullDraftSerializer): + class Meta: + model = Document + fields = [ + "id", + "name", + "rev", + "stream", + "title", + "group", + "pages", + "source_format", + "authors", + "consensus", + ] + + +class SubmittedToQueueSerializer(FullDraftSerializer): + submitted = serializers.SerializerMethodField() + consensus = serializers.SerializerMethodField() + + class Meta: + model = Document + fields = [ + "id", + "name", + "stream", + "submitted", + "consensus", + ] + + def get_submitted(self, doc) -> Optional[datetime.datetime]: + event = doc.sent_to_rfc_editor_event() + return None if event is None else event.time + + def get_consensus(self, doc) -> Optional[bool]: + return default_consensus(doc) + + +class OriginalStreamSerializer(serializers.ModelSerializer): + stream = serializers.CharField(read_only=True, source="orig_stream_id") + + class Meta: + model = Document + fields = ["rfc_number", "stream"] + + +class ReferenceSerializer(serializers.ModelSerializer): + class Meta: + model = Document + fields = ["id", "name"] + read_only_fields = ["id", "name"] + + +def _update_authors(rfc, authors_data): + # Construct unsaved instances from validated author data + new_authors = [RfcAuthor(**authdata) for authdata in authors_data] + # Update the RFC with the new author set + with transaction.atomic(): + change_events = update_rfcauthors(rfc, new_authors) + for event in change_events: + event.save() + return change_events + + +class SubseriesNameField(serializers.RegexField): + + def __init__(self, **kwargs): + # pattern: no leading 0, finite length (arbitrarily set to 5 digits) + regex = r"^(bcp|std|fyi)[1-9][0-9]{0,4}$" + super().__init__(regex, **kwargs) + + +class RfcGroupRelatedField(serializers.SlugRelatedField): + """SlugRelatedField that translates None / "" to the acronym "none" """ + + def __init__(self, **kwargs): + super().__init__( + slug_field="acronym", + queryset=Group.objects.all(), + allow_null=True, + required=False, + ) + + def run_validation(self, data=fields.empty): + # Use the Group with acronym "none" when group is not specified + if data is fields.empty or data is None or data == "": + data = "none" + return super().run_validation(data) + + +class RfcPubSerializer(serializers.ModelSerializer): + """Write-only serializer for RFC publication""" + # publication-related fields + published = serializers.DateTimeField(default_timezone=datetime.timezone.utc) + draft_name = serializers.RegexField( + required=False, regex=r"^draft-[a-zA-Z0-9-]+$" + ) + draft_rev = serializers.RegexField( + required=False, regex=r"^[0-9][0-9]$" + ) + + # fields on the RFC Document that need tweaking from ModelSerializer defaults + rfc_number = serializers.IntegerField(min_value=1, required=True) + group = RfcGroupRelatedField() + stream = serializers.PrimaryKeyRelatedField( + queryset=StreamName.objects.filter(used=True) + ) + std_level = serializers.PrimaryKeyRelatedField( + queryset=StdLevelName.objects.filter(used=True), + ) + ad = serializers.PrimaryKeyRelatedField( + queryset=Person.objects.all(), + allow_null=True, + required=False, + ) + obsoletes = serializers.SlugRelatedField( + many=True, + required=False, + slug_field="rfc_number", + queryset=Document.objects.filter(type_id="rfc"), + ) + updates = serializers.SlugRelatedField( + many=True, + required=False, + slug_field="rfc_number", + queryset=Document.objects.filter(type_id="rfc"), + ) + subseries = serializers.ListField(child=SubseriesNameField(required=False)) + # N.b., authors is _not_ a field on Document! + authors = RfcAuthorSerializer(many=True) + + class Meta: + model = Document + fields = [ + "published", + "draft_name", + "draft_rev", + "rfc_number", + "title", + "authors", + "group", + "stream", + "abstract", + "pages", + "std_level", + "ad", + "obsoletes", + "updates", + "subseries", + "keywords", + ] + + def validate(self, data): + if "draft_name" in data or "draft_rev" in data: + if "draft_name" not in data: + raise serializers.ValidationError( + {"draft_name": "Missing draft_name"}, + code="invalid-draft-spec", + ) + if "draft_rev" not in data: + raise serializers.ValidationError( + {"draft_rev": "Missing draft_rev"}, + code="invalid-draft-spec", + ) + return data + + def update(self, instance, validated_data): + raise RuntimeError("Cannot update with this serializer") + + def create(self, validated_data): + """Publish an RFC""" + published = validated_data.pop("published") + draft_name = validated_data.pop("draft_name", None) + draft_rev = validated_data.pop("draft_rev", None) + obsoletes = validated_data.pop("obsoletes", []) + updates = validated_data.pop("updates", []) + subseries = validated_data.pop("subseries", []) + + system_person = Person.objects.get(name="(System)") + + # If specified, retrieve draft and extract RFC default values from it + if draft_name is None: + draft = None + else: + # validation enforces that draft_name and draft_rev are both present + draft = Document.objects.filter( + type_id="draft", + name=draft_name, + rev=draft_rev, + ).first() + if draft is None: + raise serializers.ValidationError( + { + "draft_name": "No such draft", + "draft_rev": "No such draft", + }, + code="invalid-draft" + ) + elif draft.get_state_slug() == "rfc": + raise serializers.ValidationError( + { + "draft_name": "Draft already published as RFC", + }, + code="already-published-draft", + ) + + # Transaction to clean up if something fails + with transaction.atomic(): + # create rfc, letting validated request data override draft defaults + rfc = self._create_rfc(validated_data) + DocEvent.objects.create( + doc=rfc, + rev=rfc.rev, + type="published_rfc", + time=published, + by=system_person, + desc="RFC published", + ) + rfc.set_state(State.objects.get(used=True, type_id="rfc", slug="published")) + + # create updates / obsoletes relations + for obsoleted_rfc_pk in obsoletes: + RelatedDocument.objects.get_or_create( + source=rfc, target=obsoleted_rfc_pk, relationship_id="obs" + ) + for updated_rfc_pk in updates: + RelatedDocument.objects.get_or_create( + source=rfc, target=updated_rfc_pk, relationship_id="updates" + ) + + # create subseries relations + for subseries_doc_name in subseries: + ss_slug = subseries_doc_name[:3] + subseries_doc, ss_doc_created = Document.objects.get_or_create( + type_id=ss_slug, name=subseries_doc_name + ) + if ss_doc_created: + subseries_doc.docevent_set.create( + type=f"{ss_slug}_doc_created", + by=system_person, + desc=f"Created {subseries_doc_name} via publication of {rfc.name}", + ) + _, ss_rel_created = subseries_doc.relateddocument_set.get_or_create( + relationship_id="contains", target=rfc + ) + if ss_rel_created: + subseries_doc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Added {rfc.name} to {subseries_doc.name}", + ) + rfc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Added {rfc.name} to {subseries_doc.name}", + ) + + + # create relation with draft and update draft state + if draft is not None: + draft_changes = [] + draft_events = [] + if draft.get_state_slug() != "rfc": + draft.set_state( + State.objects.get(used=True, type="draft", slug="rfc") + ) + move_draft_files_to_archive(draft, draft.rev) + draft_changes.append(f"changed state to {draft.get_state()}") + + r, created_relateddoc = RelatedDocument.objects.get_or_create( + source=draft, target=rfc, relationship_id="became_rfc", + ) + if created_relateddoc: + change = "created {rel_name} relationship between {pretty_draft_name} and {pretty_rfc_name}".format( + rel_name=r.relationship.name.lower(), + pretty_draft_name=prettify_std_name(draft_name), + pretty_rfc_name=prettify_std_name(rfc.name), + ) + draft_changes.append(change) + + # Always set the "draft-iesg" state. This state should be set for all drafts, so + # log a warning if it is not set. What should happen here is that ietf stream + # RFCs come in as "rfcqueue" and are set to "pub" when they appear in the RFC index. + # Other stream documents should normally be "idexists" and be left that way. The + # code here *actually* leaves "draft-iesg" state alone if it is "idexists" or "pub", + # and changes any other state to "pub". If unset, it changes it to "idexists". + # This reflects historical behavior and should probably be updated, but a migration + # of existing drafts (and validation of the change) is needed before we change the + # handling. + prev_iesg_state = draft.get_state("draft-iesg") + if prev_iesg_state is None: + log.log(f'Warning while processing {rfc.name}: {draft.name} has no "draft-iesg" state') + new_iesg_state = State.objects.get(type_id="draft-iesg", slug="idexists") + elif prev_iesg_state.slug not in ("pub", "idexists"): + if prev_iesg_state.slug != "rfcqueue": + log.log( + 'Warning while processing {}: {} is in "draft-iesg" state {} (expected "rfcqueue")'.format( + rfc.name, draft.name, prev_iesg_state.slug + ) + ) + new_iesg_state = State.objects.get(type_id="draft-iesg", slug="pub") + else: + new_iesg_state = prev_iesg_state + + if new_iesg_state != prev_iesg_state: + draft.set_state(new_iesg_state) + draft_changes.append(f"changed {new_iesg_state.type.label} to {new_iesg_state}") + e = update_action_holders(draft, prev_iesg_state, new_iesg_state) + if e: + draft_events.append(e) + + # If the draft and RFC streams agree, move draft to "pub" stream state. If not, complain. + if draft.stream != rfc.stream: + log.log("Warning while processing {}: draft {} stream is {} but RFC stream is {}".format( + rfc.name, draft.name, draft.stream, rfc.stream + )) + elif draft.stream.slug in ["iab", "irtf", "ise", "editorial"]: + stream_slug = f"draft-stream-{draft.stream.slug}" + prev_state = draft.get_state(stream_slug) + if prev_state is not None and prev_state.slug != "pub": + new_state = State.objects.select_related("type").get(used=True, type__slug=stream_slug, slug="pub") + draft.set_state(new_state) + draft_changes.append( + f"changed {new_state.type.label} to {new_state}" + ) + e = update_action_holders(draft, prev_state, new_state) + if e: + draft_events.append(e) + if draft_changes: + draft_events.append( + DocEvent.objects.create( + doc=draft, + rev=draft.rev, + by=system_person, + type="sync_from_rfc_editor", + desc=f"Updated while publishing {rfc.name} ({', '.join(draft_changes)})", + ) + ) + draft.save_with_history(draft_events) + + return rfc + + def _create_rfc(self, validated_data): + authors_data = validated_data.pop("authors") + rfc = Document.objects.create( + type_id="rfc", + name=f"rfc{validated_data['rfc_number']}", + **validated_data, + ) + for order, author_data in enumerate(authors_data): + rfc.rfcauthor_set.create( + order=order, + **author_data, + ) + return rfc + + +class EditableRfcSerializer(serializers.ModelSerializer): + # Would be nice to reconcile this with ietf.doc.serializers.RfcSerializer. + # The purposes of that serializer (representing data for Red) and this one + # (accepting updates from Purple) are different enough that separate formats + # may be needed, but if not it'd be nice to have a single RfcSerializer that + # can serve both. + # + # Should also consider whether this and RfcPubSerializer should merge. + # + # Treats published and subseries fields as write-only. This isn't quite correct, + # but makes it easier and we don't currently use the serialized value except for + # debugging. + published = serializers.DateTimeField( + default_timezone=datetime.timezone.utc, + write_only=True, + ) + authors = RfcAuthorSerializer(many=True, min_length=1, source="rfcauthor_set") + subseries = serializers.ListField( + child=SubseriesNameField(required=False), + write_only=True, + ) + updates = serializers.ListField( + child=serializers.IntegerField(), + required=False, + write_only=True, + help_text="List of RFC numbers this document updates." + ) + obsoletes = serializers.ListField( + child=serializers.IntegerField(), + required=False, + write_only=True, + help_text="List of RFC numbers this document obsoletes." + ) + + class Meta: + model = Document + fields = [ + "published", + "title", + "authors", + "stream", + "abstract", + "pages", + "std_level", + "subseries", + "keywords", + "updates", + "obsoletes", + ] + + def _validate_rfc_number_list(self, field_name, rfc_numbers): + """Raise ValidationError if any RFC numbers in the list don't exist.""" + unknown = [ + n for n in rfc_numbers + if not Document.objects.filter(rfc_number=n, type_id="rfc").exists() + ] + if unknown: + raise serializers.ValidationError( + {field_name: [f"Unknown RFC number: {n}" for n in unknown]} + ) + return rfc_numbers + + def validate_updates(self, value): + return self._validate_rfc_number_list("updates", value) + + def validate_obsoletes(self, value): + return self._validate_rfc_number_list("obsoletes", value) + + def create(self, validated_data): + raise RuntimeError("Cannot create with this serializer") + + def update(self, instance, validated_data): + assert isinstance(instance, Document) + assert instance.type_id == "rfc" + rfc = instance # get better name + + system_person = Person.objects.get(name="(System)") + + # Remove data that needs special handling. Use a singleton object to detect + # missing values in case we ever support a value that needs None as an option. + omitted = object() + published = validated_data.pop("published", omitted) + subseries = validated_data.pop("subseries", omitted) + authors_data = validated_data.pop("rfcauthor_set", omitted) + updates = validated_data.pop("updates", omitted) + obsoletes = validated_data.pop("obsoletes", omitted) + + # Transaction to clean up if something fails + with transaction.atomic(): + # update the rfc Document itself + rfc_changes = [] + rfc_events = [] + + for attr, new_value in validated_data.items(): + old_value = getattr(rfc, attr) + if new_value != old_value: + rfc_changes.append( + f"changed {attr} to '{new_value}' from '{old_value}'" + ) + setattr(rfc, attr, new_value) + if len(rfc_changes) > 0: + rfc_change_summary = f"{', '.join(rfc_changes)}" + rfc_events.append( + DocEvent.objects.create( + doc=rfc, + rev=rfc.rev, + by=system_person, + type="sync_from_rfc_editor", + desc=f"Changed metadata: {rfc_change_summary}", + ) + ) + if authors_data is not omitted: + rfc_events.extend(_update_authors(instance, authors_data)) + + if published is not omitted: + published_event = rfc.latest_event(type="published_rfc") + if published_event is None: + # unexpected, but possible in theory + rfc_events.append( + DocEvent.objects.create( + doc=rfc, + rev=rfc.rev, + type="published_rfc", + time=published, + by=system_person, + desc="RFC published", + ) + ) + rfc_events.append( + DocEvent.objects.create( + doc=rfc, + rev=rfc.rev, + type="sync_from_rfc_editor", + by=system_person, + desc=( + f"Set publication timestamp to {published.isoformat()}" + ), + ) + ) + else: + original_pub_time = published_event.time + if published != original_pub_time: + published_event.time = published + published_event.save() + rfc_events.append( + DocEvent.objects.create( + doc=rfc, + rev=rfc.rev, + type="sync_from_rfc_editor", + by=system_person, + desc=( + f"Changed publication time to " + f"{published.isoformat()} from " + f"{original_pub_time.isoformat()}" + ) + ) + ) + if updates is not omitted: + RelatedDocument.objects.filter( + source=rfc, relationship_id="updates" + ).exclude(target__rfc_number__in=updates).delete() + for rfc_num in updates: + target = Document.objects.get(rfc_number=rfc_num, type_id="rfc") + RelatedDocument.objects.get_or_create( + source=rfc, relationship_id="updates", target=target + ) + if obsoletes is not omitted: + RelatedDocument.objects.filter( + source=rfc, relationship_id="obs" + ).exclude(target__rfc_number__in=obsoletes).delete() + for rfc_num in obsoletes: + target = Document.objects.get(rfc_number=rfc_num, type_id="rfc") + RelatedDocument.objects.get_or_create( + source=rfc, relationship_id="obs", target=target + ) + + # update subseries relations + if subseries is not omitted: + for subseries_doc_name in subseries: + ss_slug = subseries_doc_name[:3] + subseries_doc, ss_doc_created = Document.objects.get_or_create( + type_id=ss_slug, name=subseries_doc_name + ) + if ss_doc_created: + subseries_doc.docevent_set.create( + type=f"{ss_slug}_doc_created", + by=system_person, + desc=f"Created {subseries_doc_name} via update of {rfc.name}", + ) + _, ss_rel_created = subseries_doc.relateddocument_set.get_or_create( + relationship_id="contains", target=rfc + ) + if ss_rel_created: + subseries_doc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Added {rfc.name} to {subseries_doc.name}", + ) + rfc_events.append( + rfc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Added {rfc.name} to {subseries_doc.name}", + ) + ) + # Delete subseries relations that are no longer current + stale_subseries_relations = rfc.relations_that("contains").exclude( + source__name__in=subseries + ) + for stale_relation in stale_subseries_relations: + stale_subseries_doc = stale_relation.source + rfc_events.append( + rfc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Removed {rfc.name} from {stale_subseries_doc.name}", + ) + ) + stale_subseries_doc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Removed {rfc.name} from {stale_subseries_doc.name}", + ) + stale_subseries_relations.delete() + if len(rfc_events) > 0: + rfc.save_with_history(rfc_events) + # Gather obs and updates in both directions as a title/author change to + # this doc affects the info rendering of all of the other RFCs + needs_updating = sorted( + [ + d.rfc_number + for d in [rfc] + + rfc.related_that_doc(("obs", "updates")) + + rfc.related_that(("obs", "updates")) + ] + ) + trigger_red_precomputer_task.delay(rfc_number_list=needs_updating) + # Update the search index also + update_rfc_searchindex_task.delay(rfc.rfc_number) + return rfc + + +class RfcFileSerializer(serializers.Serializer): + # The structure of this serializer is constrained by what openapi-generator-cli's + # python generator can correctly serialize as multipart/form-data. It does not + # handle nested serializers well (or perhaps at all). ListFields with child + # ChoiceField or RegexField do not serialize correctly. DictFields don't seem + # to work. + # + # It does seem to correctly send filenames along with FileFields, even as a child + # in a ListField, so we use that to convey the file format of each item. There + # are other options we could consider (e.g., a structured CharField) but this + # works. + allowed_extensions = ( + ".html", + ".json", + ".notprepped.xml", + ".pdf", + ".txt", + ".xml", + ) + + rfc = serializers.SlugRelatedField( + slug_field="rfc_number", + queryset=Document.objects.filter(type_id="rfc"), + help_text="RFC number to which the contents belong", + ) + contents = serializers.ListField( + child=serializers.FileField( + allow_empty_file=False, + use_url=False, + ), + help_text=( + "List of content files. Filename extensions are used to identify " + "file types, but filenames are otherwise ignored." + ), + ) + mtime = serializers.DateTimeField( + required=False, + default=timezone.now, + default_timezone=datetime.UTC, + help_text="Modification timestamp to apply to uploaded files", + ) + replace = serializers.BooleanField( + required=False, + default=False, + help_text=( + "Replace existing files for this RFC. Defaults to false. When false, " + "if _any_ files already exist for the specified RFC the upload will be " + "rejected regardless of which files are being uploaded. When true," + "existing files will be removed and new ones will be put in place. BE" + "VERY CAREFUL WITH THIS OPTION IN PRODUCTION." + ), + ) + + def validate_contents(self, data): + found_extensions = [] + for uploaded_file in data: + if not hasattr(uploaded_file, "name"): + raise serializers.ValidationError( + "filename not specified for uploaded file", + code="missing-filename", + ) + ext = "".join(Path(uploaded_file.name).suffixes) + if ext not in self.allowed_extensions: + raise serializers.ValidationError( + f"File uploaded with invalid extension '{ext}'", + code="invalid-filename-ext", + ) + if ext in found_extensions: + raise serializers.ValidationError( + f"More than one file uploaded with extension '{ext}'", + code="duplicate-filename-ext", + ) + return data + + +class NotificationAckSerializer(serializers.Serializer): + message = serializers.CharField(default="ack") diff --git a/ietf/api/tests.py b/ietf/api/tests.py index 7cc56c2d0a..2a44791a5c 100644 --- a/ietf/api/tests.py +++ b/ietf/api/tests.py @@ -1,10 +1,11 @@ # Copyright The IETF Trust 2015-2024, All Rights Reserved # -*- coding: utf-8 -*- import base64 +import copy import datetime import json import html -import mock +from unittest import mock import os import sys @@ -31,16 +32,16 @@ from ietf.doc.factories import IndividualDraftFactory, WgDraftFactory, WgRfcFactory from ietf.group.factories import RoleFactory from ietf.meeting.factories import MeetingFactory, SessionFactory -from ietf.meeting.models import Session +from ietf.meeting.models import Session, Registration from ietf.nomcom.models import Volunteer from ietf.nomcom.factories import NomComFactory, nomcom_kwargs_for_year from ietf.person.factories import PersonFactory, random_faker, EmailFactory, PersonalApiKeyFactory from ietf.person.models import Email, User -from ietf.stats.models import MeetingRegistration from ietf.utils.mail import empty_outbox, outbox, get_payload_text from ietf.utils.models import DumpInfo from ietf.utils.test_utils import TestCase, login_testing_unauthorized, reload_db_objects +from . import Serializer from .ietf_utils import is_valid_token, requires_api_token from .views import EmailIngestionError @@ -49,6 +50,7 @@ 'ietf.secr.proceedings', 'ietf.ipr', 'ietf.status', + 'ietf.blobdb', ) class CustomApiTests(TestCase): @@ -460,12 +462,12 @@ def test_api_add_session_attendees(self): self.assertTrue(session.attended_set.filter(person=recman).exists()) self.assertEqual( session.attended_set.get(person=recman).time, - datetime.datetime(2023, 9, 3, 12, 34, 56, tzinfo=datetime.timezone.utc), + datetime.datetime(2023, 9, 3, 12, 34, 56, tzinfo=datetime.UTC), ) self.assertTrue(session.attended_set.filter(person=otherperson).exists()) self.assertEqual( session.attended_set.get(person=otherperson).time, - datetime.datetime(2023, 9, 3, 3, 0, 19, tzinfo=datetime.timezone.utc), + datetime.datetime(2023, 9, 3, 3, 0, 19, tzinfo=datetime.UTC), ) def test_api_upload_polls_and_chatlog(self): @@ -703,100 +705,146 @@ def test_api_v2_person_export_view(self): self.assertEqual(data['ascii'], robot.ascii) self.assertEqual(data['user']['email'], robot.user.email) - def test_api_new_meeting_registration(self): + @override_settings(APP_API_TOKENS={"ietf.api.views.api_new_meeting_registration_v2": ["valid-token"]}) + def test_api_new_meeting_registration_v2(self): meeting = MeetingFactory(type_id='ietf') - reg = { - 'apikey': 'invalid', + person = PersonFactory() + reg_detail = { + 'email': person.email().address, + 'first_name': person.first_name(), + 'last_name': person.last_name(), + 'meeting': meeting.number, 'affiliation': "Alguma Corporação", 'country_code': 'PT', - 'email': 'foo@example.pt', - 'first_name': 'Foo', - 'last_name': 'Bar', - 'meeting': meeting.number, - 'reg_type': 'hackathon', - 'ticket_type': '', - 'checkedin': 'False', - 'is_nomcom_volunteer': 'False', + 'checkedin': False, + 'is_nomcom_volunteer': False, + 'cancelled': False, + 'tickets': [{'attendance_type': 'onsite', 'ticket_type': 'week_pass'}], } - url = urlreverse('ietf.api.views.api_new_meeting_registration') - r = self.client.post(url, reg) - self.assertContains(r, 'Invalid apikey', status_code=403) - oidcp = PersonFactory(user__is_staff=True) - # Make sure 'oidcp' has an acceptable role - RoleFactory(name_id='robot', person=oidcp, email=oidcp.email(), group__acronym='secretariat') - key = PersonalApiKeyFactory(person=oidcp, endpoint=url) - reg['apikey'] = key.hash() - # - # Test valid POST - # FIXME: sometimes, there seems to be something in the outbox? - old_len = len(outbox) - r = self.client.post(url, reg) - self.assertContains(r, "Accepted, New registration, Email sent", status_code=202) - # - # Check outgoing mail - self.assertEqual(len(outbox), old_len + 1) - body = get_payload_text(outbox[-1]) - self.assertIn(reg['email'], outbox[-1]['To'] ) - self.assertIn(reg['email'], body) - self.assertIn('account creation request', body) + reg_data = {'objects': {reg_detail['email']: reg_detail}} + url = urlreverse('ietf.api.views.api_new_meeting_registration_v2') # - # Check record - obj = MeetingRegistration.objects.get(email=reg['email'], meeting__number=reg['meeting']) - for key in ['affiliation', 'country_code', 'first_name', 'last_name', 'person', 'reg_type', 'ticket_type', 'checkedin']: - self.assertEqual(getattr(obj, key), False if key=='checkedin' else reg.get(key) , "Bad data for field '%s'" % key) + # Test invalid key + r = self.client.post(url, data=json.dumps(reg_data), content_type='application/json', headers={"X-Api-Key": "invalid-token"}) + self.assertEqual(r.status_code, 403) # - # Test with existing user - person = PersonFactory() - reg['email'] = person.email().address - reg['first_name'] = person.first_name() - reg['last_name'] = person.last_name() + # Test invalid data + bad_reg_data = copy.deepcopy(reg_data) + del bad_reg_data['objects'][reg_detail['email']]['email'] + r = self.client.post(url, data=json.dumps(bad_reg_data), content_type='application/json', headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 400) # - r = self.client.post(url, reg) - self.assertContains(r, "Accepted, New registration", status_code=202) + # Test valid POST + r = self.client.post(url, data=json.dumps(reg_data), content_type='application/json', headers={"X-Api-Key": "valid-token"}) + self.assertContains(r, "Success", status_code=202) # - # There should be no new outgoing mail - self.assertEqual(len(outbox), old_len + 1) + # Check record + objects = Registration.objects.filter(email=reg_detail['email'], meeting__number=reg_detail['meeting']) + self.assertEqual(objects.count(), 1) + obj = objects[0] + for key in ['affiliation', 'country_code', 'first_name', 'last_name', 'checkedin']: + self.assertEqual(getattr(obj, key), False if key == 'checkedin' else reg_detail.get(key), f"Bad data for field {key}") + self.assertEqual(obj.tickets.count(), 1) + ticket = obj.tickets.first() + self.assertEqual(ticket.ticket_type.slug, reg_detail['tickets'][0]['ticket_type']) + self.assertEqual(ticket.attendance_type.slug, reg_detail['tickets'][0]['attendance_type']) + self.assertEqual(obj.person, person) # - # Test multiple reg types - reg['reg_type'] = 'remote' - reg['ticket_type'] = 'full_week_pass' - r = self.client.post(url, reg) - self.assertContains(r, "Accepted, New registration", status_code=202) - objs = MeetingRegistration.objects.filter(email=reg['email'], meeting__number=reg['meeting']) - self.assertEqual(len(objs), 2) - self.assertEqual(objs.filter(reg_type='hackathon').count(), 1) - self.assertEqual(objs.filter(reg_type='remote', ticket_type='full_week_pass').count(), 1) - self.assertEqual(len(outbox), old_len + 1) + # Test update (switch to remote) + reg_detail = { + 'affiliation': "Alguma Corporação", + 'country_code': 'PT', + 'email': person.email().address, + 'first_name': person.first_name(), + 'last_name': person.last_name(), + 'meeting': meeting.number, + 'checkedin': False, + 'is_nomcom_volunteer': False, + 'cancelled': False, + 'tickets': [{'attendance_type': 'remote', 'ticket_type': 'week_pass'}], + } + reg_data = {'objects': {reg_detail['email']: reg_detail}} + r = self.client.post(url, data=json.dumps(reg_data), content_type='application/json', headers={"X-Api-Key": "valid-token"}) + self.assertContains(r, "Success", status_code=202) + objects = Registration.objects.filter(email=reg_detail['email'], meeting__number=reg_detail['meeting']) + self.assertEqual(objects.count(), 1) + obj = objects[0] + self.assertEqual(obj.tickets.count(), 1) + ticket = obj.tickets.first() + self.assertEqual(ticket.ticket_type.slug, reg_detail['tickets'][0]['ticket_type']) + self.assertEqual(ticket.attendance_type.slug, reg_detail['tickets'][0]['attendance_type']) # - # Test incomplete POST - drop_fields = ['affiliation', 'first_name', 'reg_type'] - for field in drop_fields: - del reg[field] - r = self.client.post(url, reg) - self.assertContains(r, 'Missing parameters:', status_code=400) - err, fields = r.content.decode().split(':', 1) - missing_fields = [f.strip() for f in fields.split(',')] - self.assertEqual(set(missing_fields), set(drop_fields)) - - def test_api_new_meeting_registration_nomcom_volunteer(self): - '''Test that Volunteer is created if is_nomcom_volunteer=True - is submitted to API - ''' - meeting = MeetingFactory(type_id='ietf') - reg = { - 'apikey': 'invalid', + # Test multiple + reg_detail = { 'affiliation': "Alguma Corporação", 'country_code': 'PT', + 'email': person.email().address, + 'first_name': person.first_name(), + 'last_name': person.last_name(), 'meeting': meeting.number, - 'reg_type': 'onsite', - 'ticket_type': '', - 'checkedin': 'False', - 'is_nomcom_volunteer': 'False', + 'checkedin': False, + 'is_nomcom_volunteer': False, + 'cancelled': False, + 'tickets': [ + {'attendance_type': 'onsite', 'ticket_type': 'one_day'}, + {'attendance_type': 'remote', 'ticket_type': 'week_pass'}, + ], } + reg_data = {'objects': {reg_detail['email']: reg_detail}} + r = self.client.post(url, data=json.dumps(reg_data), content_type='application/json', headers={"X-Api-Key": "valid-token"}) + self.assertContains(r, "Success", status_code=202) + objects = Registration.objects.filter(email=reg_detail['email'], meeting__number=reg_detail['meeting']) + self.assertEqual(objects.count(), 1) + obj = objects[0] + self.assertEqual(obj.tickets.count(), 2) + self.assertEqual(obj.tickets.filter(attendance_type__slug='onsite').count(), 1) + self.assertEqual(obj.tickets.filter(attendance_type__slug='remote').count(), 1) + + @override_settings(APP_API_TOKENS={"ietf.api.views.api_new_meeting_registration_v2": ["valid-token"]}) + def test_api_new_meeting_registration_v2_cancelled(self): + meeting = MeetingFactory(type_id='ietf') + person = PersonFactory() + reg_detail = { + 'affiliation': "Acme", + 'country_code': 'US', + 'email': person.email().address, + 'first_name': person.first_name(), + 'last_name': person.last_name(), + 'meeting': meeting.number, + 'checkedin': False, + 'is_nomcom_volunteer': False, + 'cancelled': False, + 'tickets': [{'attendance_type': 'onsite', 'ticket_type': 'week_pass'}], + } + reg_data = {'objects': {reg_detail['email']: reg_detail}} + url = urlreverse('ietf.api.views.api_new_meeting_registration_v2') + self.assertEqual(Registration.objects.count(), 0) + r = self.client.post(url, data=json.dumps(reg_data), content_type='application/json', headers={"X-Api-Key": "valid-token"}) + self.assertContains(r, "Success", status_code=202) + self.assertEqual(Registration.objects.count(), 1) + reg_detail['cancelled'] = True + r = self.client.post(url, data=json.dumps(reg_data), content_type='application/json', headers={"X-Api-Key": "valid-token"}) + self.assertContains(r, "Success", status_code=202) + self.assertEqual(Registration.objects.count(), 0) + + @override_settings(APP_API_TOKENS={"ietf.api.views.api_new_meeting_registration_v2": ["valid-token"]}) + def test_api_new_meeting_registration_v2_nomcom(self): + meeting = MeetingFactory(type_id='ietf') person = PersonFactory() - reg['email'] = person.email().address - reg['first_name'] = person.first_name() - reg['last_name'] = person.last_name() + reg_detail = { + 'affiliation': "Acme", + 'country_code': 'US', + 'email': person.email().address, + 'first_name': person.first_name(), + 'last_name': person.last_name(), + 'meeting': meeting.number, + 'checkedin': False, + 'is_nomcom_volunteer': False, + 'cancelled': False, + 'tickets': [{'attendance_type': 'onsite', 'ticket_type': 'week_pass'}], + } + reg_data = {'objects': {reg_detail['email']: reg_detail}} + url = urlreverse('ietf.api.views.api_new_meeting_registration_v2') now = datetime.datetime.now() if now.month > 10: year = now.year + 1 @@ -804,23 +852,17 @@ def test_api_new_meeting_registration_nomcom_volunteer(self): year = now.year # create appropriate group and nomcom objects nomcom = NomComFactory.create(is_accepting_volunteers=True, **nomcom_kwargs_for_year(year)) - url = urlreverse('ietf.api.views.api_new_meeting_registration') - oidcp = PersonFactory(user__is_staff=True) - # Make sure 'oidcp' has an acceptable role - RoleFactory(name_id='robot', person=oidcp, email=oidcp.email(), group__acronym='secretariat') - key = PersonalApiKeyFactory(person=oidcp, endpoint=url) - reg['apikey'] = key.hash() # first test is_nomcom_volunteer False - r = self.client.post(url, reg) - self.assertContains(r, "Accepted, New registration", status_code=202) + r = self.client.post(url, data=json.dumps(reg_data), content_type='application/json', headers={"X-Api-Key": "valid-token"}) + self.assertContains(r, "Success", status_code=202) # assert no Volunteers exists self.assertEqual(Volunteer.objects.count(), 0) # test is_nomcom_volunteer True - reg['is_nomcom_volunteer'] = 'True' - r = self.client.post(url, reg) - self.assertContains(r, "Accepted, Updated registration", status_code=202) + reg_detail['is_nomcom_volunteer'] = True + r = self.client.post(url, data=json.dumps(reg_data), content_type='application/json', headers={"X-Api-Key": "valid-token"}) + self.assertContains(r, "Success", status_code=202) # assert Volunteer exists self.assertEqual(Volunteer.objects.count(), 1) volunteer = Volunteer.objects.last() @@ -829,7 +871,7 @@ def test_api_new_meeting_registration_nomcom_volunteer(self): self.assertEqual(volunteer.origin, 'registration') def test_api_version(self): - DumpInfo.objects.create(date=timezone.datetime(2022,8,31,7,10,1,tzinfo=datetime.timezone.utc), host='testapi.example.com',tz='UTC') + DumpInfo.objects.create(date=timezone.datetime(2022,8,31,7,10,1,tzinfo=datetime.UTC), host='testapi.example.com',tz='UTC') url = urlreverse('ietf.api.views.version') r = self.client.get(url) data = r.json() @@ -973,6 +1015,39 @@ def test_active_email_list(self): self.assertCountEqual(result.keys(), ["addresses"]) self.assertCountEqual(result["addresses"], Email.objects.filter(active=True).values_list("address", flat=True)) + @override_settings(APP_API_TOKENS={"ietf.api.views.related_email_list": ["valid-token"]}) + def test_related_email_list(self): + joe = EmailFactory(address='joe@work.com') + EmailFactory(address='joe@home.com', person=joe.person) + EmailFactory(address='jòe@spain.com', person=joe.person) + url = urlreverse("ietf.api.views.related_email_list", kwargs={'email': 'joe@home.com'}) + # no api key + r = self.client.get(url, headers={}) + self.assertEqual(r.status_code, 403) + # invalid api key + r = self.client.get(url, headers={"X-Api-Key": "not-the-valid-token"}) + self.assertEqual(r.status_code, 403) + # wrong method + r = self.client.post(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 405) + # valid + r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 200) + self.assertEqual(r.headers["Content-Type"], "application/json") + result = json.loads(r.content) + self.assertCountEqual(result.keys(), ["addresses"]) + self.assertCountEqual(result["addresses"], joe.person.email_set.values_list("address", flat=True)) + # non-ascii + non_ascii_url = urlreverse("ietf.api.views.related_email_list", kwargs={'email': 'jòe@spain.com'}) + r = self.client.get(non_ascii_url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 200) + result = json.loads(r.content) + self.assertTrue('joe@home.com' in result["addresses"]) + # email not found + not_found_url = urlreverse("ietf.api.views.related_email_list", kwargs={'email': 'nobody@nowhere.com'}) + r = self.client.get(not_found_url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 404) + @override_settings(APP_API_TOKENS={"ietf.api.views.role_holder_addresses": ["valid-token"]}) def test_role_holder_addresses(self): url = urlreverse("ietf.api.views.role_holder_addresses") @@ -1382,7 +1457,7 @@ def test_bad_post(self): data = self.response_data(r) self.assertEqual(data["result"], "failure") self.assertEqual(data["reason"], "invalid post") - + bad = dict(authtoken=self.valid_token, username=self.valid_person.user.username, password=self.valid_password) r = self.client.post(self.url, bad) self.assertEqual(r.status_code, 200) @@ -1390,8 +1465,9 @@ def test_bad_post(self): self.assertEqual(data["result"], "failure") self.assertEqual(data["reason"], "invalid post") + @override_settings() def test_notokenstore(self): - self.assertFalse(hasattr(settings, "APP_API_TOKENS")) + del settings.APP_API_TOKENS # only affects overridden copy of settings! r = self.client.post(self.url,self.valid_body_with_good_password) self.assertEqual(r.status_code, 200) data = self.response_data(r) @@ -1421,7 +1497,7 @@ def test_good_password(self): data = self.response_data(r) self.assertEqual(data["result"], "success") -class TastypieApiTestCase(ResourceTestCaseMixin, TestCase): +class TastypieApiTests(ResourceTestCaseMixin, TestCase): def __init__(self, *args, **kwargs): self.apps = {} for app_name in settings.INSTALLED_APPS: @@ -1431,7 +1507,7 @@ def __init__(self, *args, **kwargs): models_path = os.path.join(os.path.dirname(app.__file__), "models.py") if os.path.exists(models_path): self.apps[name] = app_name - super(TastypieApiTestCase, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def test_api_top_level(self): client = Client(Accept='application/json') @@ -1466,6 +1542,21 @@ def test_all_model_resources_exist(self): self.assertIn(model._meta.model_name, list(app_resources.keys()), "There doesn't seem to be any API resource for model %s.models.%s"%(app.__name__,model.__name__,)) + def test_serializer_to_etree_handles_nulls(self): + """Serializer to_etree() should handle a null character""" + serializer = Serializer() + try: + serializer.to_etree("string with no nulls in it") + except ValueError: + self.fail("serializer.to_etree raised ValueError on an ordinary string") + try: + serializer.to_etree("string with a \x00 in it") + except ValueError: + self.fail( + "serializer.to_etree raised ValueError on a string " + "containing a null character" + ) + class RfcdiffSupportTests(TestCase): diff --git a/ietf/api/tests_ietf_utils.py b/ietf/api/tests_ietf_utils.py new file mode 100644 index 0000000000..b8d7fea7b4 --- /dev/null +++ b/ietf/api/tests_ietf_utils.py @@ -0,0 +1,86 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.test import RequestFactory +from django.test.utils import override_settings + +from ietf.api.ietf_utils import is_valid_token, requires_api_token +from ietf.utils.test_utils import TestCase + + +class IetfUtilsTests(TestCase): + @override_settings( + APP_API_TOKENS={ + "ietf.api.foobar": ["valid-token"], + "ietf.api.misconfigured": "valid-token", # misconfigured + } + ) + def test_is_valid_token(self): + self.assertFalse(is_valid_token("ietf.fake.endpoint", "valid-token")) + self.assertFalse(is_valid_token("ietf.api.foobar", "invalid-token")) + self.assertFalse(is_valid_token("ietf.api.foobar", None)) + self.assertTrue(is_valid_token("ietf.api.foobar", "valid-token")) + + # misconfiguration + self.assertFalse(is_valid_token("ietf.api.misconfigured", "v")) + self.assertFalse(is_valid_token("ietf.api.misconfigured", None)) + self.assertTrue(is_valid_token("ietf.api.misconfigured", "valid-token")) + + @override_settings( + APP_API_TOKENS={ + "ietf.api.foo": ["valid-token"], + "ietf.api.bar": ["another-token"], + "ietf.api.misconfigured": "valid-token", # misconfigured + } + ) + def test_requires_api_token(self): + @requires_api_token("ietf.api.foo") + def protected_function(request): + return f"Access granted: {request.method}" + + # request with a valid token + request = RequestFactory().get( + "/some/url", headers={"X_API_KEY": "valid-token"} + ) + result = protected_function(request) + self.assertEqual(result, "Access granted: GET") + + # request with an invalid token + request = RequestFactory().get( + "/some/url", headers={"X_API_KEY": "invalid-token"} + ) + result = protected_function(request) + self.assertEqual(result.status_code, 403) + + # request without a token + request = RequestFactory().get("/some/url", headers={"X_API_KEY": ""}) + result = protected_function(request) + self.assertEqual(result.status_code, 403) + + # request without a X_API_KEY token + request = RequestFactory().get("/some/url") + result = protected_function(request) + self.assertEqual(result.status_code, 403) + + # request with a valid token for another API endpoint + request = RequestFactory().get( + "/some/url", headers={"X_API_KEY": "another-token"} + ) + result = protected_function(request) + self.assertEqual(result.status_code, 403) + + # requests for a misconfigured endpoint + @requires_api_token("ietf.api.misconfigured") + def another_protected_function(request): + return f"Access granted: {request.method}" + + # request with valid token + request = RequestFactory().get( + "/some/url", headers={"X_API_KEY": "valid-token"} + ) + result = another_protected_function(request) + self.assertEqual(result, "Access granted: GET") + + # request with invalid token with the correct initial character + request = RequestFactory().get("/some/url", headers={"X_API_KEY": "v"}) + result = another_protected_function(request) + self.assertEqual(result.status_code, 403) diff --git a/ietf/api/tests_serializers_rpc.py b/ietf/api/tests_serializers_rpc.py new file mode 100644 index 0000000000..5151f337d5 --- /dev/null +++ b/ietf/api/tests_serializers_rpc.py @@ -0,0 +1,241 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from unittest import mock + +from django.utils import timezone + +from ietf.utils.test_utils import TestCase +from ietf.doc.models import Document +from ietf.doc.factories import WgRfcFactory +from .serializers_rpc import EditableRfcSerializer + + +class EditableRfcSerializerTests(TestCase): + def test_create(self): + serializer = EditableRfcSerializer( + data={ + "published": timezone.now(), + "title": "Yadda yadda yadda", + "authors": [ + { + "titlepage_name": "B. Fett", + "is_editor": False, + "affiliation": "DBA Galactic Empire", + "country": "", + }, + ], + "stream": "ietf", + "abstract": "A long time ago in a galaxy far, far away...", + "pages": 3, + "std_level": "inf", + "subseries": ["fyi999"], + } + ) + self.assertTrue(serializer.is_valid()) + with self.assertRaises(RuntimeError, msg="serializer does not allow create()"): + serializer.save() + + @mock.patch("ietf.api.serializers_rpc.update_rfc_searchindex_task") + @mock.patch("ietf.api.serializers_rpc.trigger_red_precomputer_task") + def test_update(self, mock_trigger_red_task, mock_update_searchindex_task): + updates = WgRfcFactory.create_batch(2) + obsoletes = WgRfcFactory.create_batch(2) + rfc = WgRfcFactory(pages=10) + updated_by = WgRfcFactory.create_batch(2) + obsoleted_by = WgRfcFactory.create_batch(2) + for d in updates: + rfc.relateddocument_set.create(relationship_id="updates",target=d) + for d in obsoletes: + rfc.relateddocument_set.create(relationship_id="updates",target=d) + for d in updated_by: + d.relateddocument_set.create(relationship_id="updates",target=rfc) + for d in obsoleted_by: + d.relateddocument_set.create(relationship_id="updates",target=rfc) + serializer = EditableRfcSerializer( + instance=rfc, + data={ + "published": timezone.now(), + "title": "Yadda yadda yadda", + "authors": [ + { + "titlepage_name": "B. Fett", + "is_editor": False, + "affiliation": "DBA Galactic Empire", + "country": "", + }, + ], + "stream": "ise", + "abstract": "A long time ago in a galaxy far, far away...", + "pages": 3, + "std_level": "inf", + "subseries": ["fyi999"], + }, + ) + self.assertTrue(serializer.is_valid()) + result = serializer.save() + result.refresh_from_db() + self.assertEqual(result.title, "Yadda yadda yadda") + self.assertEqual( + list( + result.rfcauthor_set.values( + "titlepage_name", "is_editor", "affiliation", "country" + ) + ), + [ + { + "titlepage_name": "B. Fett", + "is_editor": False, + "affiliation": "DBA Galactic Empire", + "country": "", + }, + ], + ) + self.assertEqual(result.stream_id, "ise") + self.assertEqual( + result.abstract, "A long time ago in a galaxy far, far away..." + ) + self.assertEqual(result.pages, 3) + self.assertEqual(result.std_level_id, "inf") + self.assertEqual( + result.part_of(), + [Document.objects.get(name="fyi999")], + ) + # Confirm that red precomputer was triggered correctly + self.assertTrue(mock_trigger_red_task.delay.called) + _, mock_kwargs = mock_trigger_red_task.delay.call_args + self.assertIn("rfc_number_list", mock_kwargs) + expected_numbers = sorted( + [ + d.rfc_number + for d in [rfc] + updates + obsoletes + updated_by + obsoleted_by + ] + ) + self.assertEqual(mock_kwargs["rfc_number_list"], expected_numbers) + # Confirm that the search index update task was triggered correctly + self.assertTrue(mock_update_searchindex_task.delay.called) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) + + @mock.patch("ietf.api.serializers_rpc.update_rfc_searchindex_task") + @mock.patch("ietf.api.serializers_rpc.trigger_red_precomputer_task") + def test_partial_update(self, mock_trigger_red_task, mock_update_searchindex_task): + # We could test other permutations of fields, but authors is a partial update + # we know we are going to use, so verifying that one in particular. + updates = WgRfcFactory.create_batch(2) + obsoletes = WgRfcFactory.create_batch(2) + rfc = WgRfcFactory(pages=10, abstract="do or do not", title="padawan") + updated_by = WgRfcFactory.create_batch(2) + obsoleted_by = WgRfcFactory.create_batch(2) + for d in updates: + rfc.relateddocument_set.create(relationship_id="updates",target=d) + for d in obsoletes: + rfc.relateddocument_set.create(relationship_id="updates",target=d) + for d in updated_by: + d.relateddocument_set.create(relationship_id="updates",target=rfc) + for d in obsoleted_by: + d.relateddocument_set.create(relationship_id="updates",target=rfc) + serializer = EditableRfcSerializer( + partial=True, + instance=rfc, + data={ + "authors": [ + { + "titlepage_name": "B. Fett", + "is_editor": False, + "affiliation": "DBA Galactic Empire", + "country": "", + }, + ], + }, + ) + self.assertTrue(serializer.is_valid()) + result = serializer.save() + result.refresh_from_db() + self.assertEqual(rfc.title, "padawan") + self.assertEqual( + list( + result.rfcauthor_set.values( + "titlepage_name", "is_editor", "affiliation", "country" + ) + ), + [ + { + "titlepage_name": "B. Fett", + "is_editor": False, + "affiliation": "DBA Galactic Empire", + "country": "", + }, + ], + ) + self.assertEqual(result.stream_id, "ietf") + self.assertEqual(result.abstract, "do or do not") + self.assertEqual(result.pages, 10) + self.assertEqual(result.std_level_id, "ps") + self.assertEqual(result.part_of(), []) + # Confirm that the red precomputer was triggered correctly + self.assertTrue(mock_trigger_red_task.delay.called) + _, mock_kwargs = mock_trigger_red_task.delay.call_args + self.assertIn("rfc_number_list", mock_kwargs) + expected_numbers = sorted( + [ + d.rfc_number + for d in [rfc] + updates + obsoletes + updated_by + obsoleted_by + ] + ) + self.assertEqual(mock_kwargs["rfc_number_list"], expected_numbers) + # Confirm that the search index update task was called correctly + self.assertTrue(mock_update_searchindex_task.delay.called) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) + + # Test only a field on the Document itself to be sure that it works + mock_trigger_red_task.delay.reset_mock() + mock_update_searchindex_task.delay.reset_mock() + serializer = EditableRfcSerializer( + partial=True, + instance=rfc, + data={"title": "jedi master"}, + ) + self.assertTrue(serializer.is_valid()) + result = serializer.save() + result.refresh_from_db() + self.assertEqual(rfc.title, "jedi master") + # Confirm that the red precomputer was triggered correctly + self.assertTrue(mock_trigger_red_task.delay.called) + _, mock_kwargs = mock_trigger_red_task.delay.call_args + self.assertIn("rfc_number_list", mock_kwargs) + self.assertEqual(mock_kwargs["rfc_number_list"], expected_numbers) + # Confirm that the search index update task was called correctly + self.assertTrue(mock_update_searchindex_task.delay.called) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) + + def test_unknown_rfc_number_rejected(self): + """Unknown RFC numbers in updates/obsoletes should cause validation failure.""" + from django.db.models import Max + + rfc = WgRfcFactory() + unknown_rfc_number = ( + Document.objects.filter(rfc_number__isnull=False).aggregate( + m=Max("rfc_number") + 1 + )["m"] + or 10000 + ) + + for field in ("updates", "obsoletes"): + serializer = EditableRfcSerializer( + partial=True, + instance=rfc, + data={field: [unknown_rfc_number]}, + ) + self.assertFalse( + serializer.is_valid(), + msg=f"{field} with unknown RFC number should be invalid", + ) + self.assertIn(field, serializer.errors) diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py new file mode 100644 index 0000000000..6d147c00b0 --- /dev/null +++ b/ietf/api/tests_views_rpc.py @@ -0,0 +1,567 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +import datetime +from io import StringIO +from pathlib import Path + +from django.conf import settings +from django.core.files.base import ContentFile +from django.db.models import Max +from django.db.models.functions import Coalesce +from django.test.utils import override_settings +from django.urls import reverse as urlreverse +import mock +from django.utils import timezone + +from ietf.api.views_rpc import DestinationHelperMixin +from ietf.blobdb.models import Blob +from ietf.doc.factories import ( + IndividualDraftFactory, + RfcFactory, + WgDraftFactory, + WgRfcFactory, +) +from ietf.doc.models import RelatedDocument, Document +from ietf.group.factories import RoleFactory, GroupFactory +from ietf.person.factories import PersonFactory +from ietf.sync.rfcindex import rfcindex_is_dirty +from ietf.utils.models import DirtyBits +from ietf.utils.test_utils import APITestCase, reload_db_objects + + +class RpcApiTests(APITestCase): + @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + def test_draftviewset_references(self): + viewname = "ietf.api.purple_api.draft-references" + + # non-existent draft + bad_id = Document.objects.aggregate(unused_id=Coalesce(Max("id"), 0) + 100)[ + "unused_id" + ] + url = urlreverse(viewname, kwargs={"doc_id": bad_id}) + # Without credentials + r = self.client.get(url) + self.assertEqual(r.status_code, 403) + # Add credentials + r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 404) + + # draft without any normative references + draft = IndividualDraftFactory() + draft = reload_db_objects(draft) + url = urlreverse(viewname, kwargs={"doc_id": draft.id}) + r = self.client.get(url) + self.assertEqual(r.status_code, 403) + r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 200) + refs = r.json() + self.assertEqual(refs, []) + + # draft without any normative references but with an informative reference + draft_foo = IndividualDraftFactory() + draft_foo = reload_db_objects(draft_foo) + RelatedDocument.objects.create( + source=draft, target=draft_foo, relationship_id="refinfo" + ) + url = urlreverse(viewname, kwargs={"doc_id": draft.id}) + r = self.client.get(url) + self.assertEqual(r.status_code, 403) + r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 200) + refs = r.json() + self.assertEqual(refs, []) + + # draft with a normative reference + draft_bar = IndividualDraftFactory() + draft_bar = reload_db_objects(draft_bar) + RelatedDocument.objects.create( + source=draft, target=draft_bar, relationship_id="refnorm" + ) + url = urlreverse(viewname, kwargs={"doc_id": draft.id}) + r = self.client.get(url) + self.assertEqual(r.status_code, 403) + r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 200) + refs = r.json() + self.assertEqual(len(refs), 1) + self.assertEqual(refs[0]["id"], draft_bar.id) + self.assertEqual(refs[0]["name"], draft_bar.name) + + @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + @mock.patch("ietf.doc.tasks.signal_update_rfc_metadata_task.delay") + def test_notify_rfc_published(self, mock_task_delay): + url = urlreverse("ietf.api.purple_api.notify_rfc_published") + area = GroupFactory(type_id="area") + rfc_group = GroupFactory(type_id="wg") + draft_ad = RoleFactory(group=area, name_id="ad").person + rfc_ad = PersonFactory() + draft_authors = PersonFactory.create_batch(2) + rfc_authors = PersonFactory.create_batch(3) + draft = WgDraftFactory( + group__parent=area, authors=draft_authors, ad=draft_ad, stream_id="ietf" + ) + rfc_stream_id = "ise" + assert isinstance(draft, Document), "WgDraftFactory should generate a Document" + updates = RfcFactory.create_batch(2) + obsoletes = RfcFactory.create_batch(2) + unused_rfc_number = ( + Document.objects.filter(rfc_number__isnull=False).aggregate( + unused_rfc_number=Max("rfc_number") + 1 + )["unused_rfc_number"] + or 10000 + ) + + post_data = { + "published": "2025-12-17T20:29:00Z", + "draft_name": draft.name, + "draft_rev": draft.rev, + "rfc_number": unused_rfc_number, + "title": "RFC " + draft.title, + "authors": [ + { + "titlepage_name": f"titlepage {author.name}", + "is_editor": False, + "person": author.pk, + "email": author.email_address(), + "affiliation": "Some Affiliation", + "country": "CA", + } + for author in rfc_authors + ], + "group": rfc_group.acronym, + "stream": rfc_stream_id, + "abstract": "RFC version of " + draft.abstract, + "pages": draft.pages + 10, + "std_level": "ps", + "ad": rfc_ad.pk, + "obsoletes": [o.rfc_number for o in obsoletes], + "updates": [o.rfc_number for o in updates], + "subseries": [], + } + r = self.client.post(url, data=post_data, format="json") + self.assertEqual(r.status_code, 403) + + # Put a file in the way. Post should fail because files exists + rfc_path = Path(settings.RFC_PATH) + (rfc_path / "prerelease").mkdir() + file_in_the_way = rfc_path / f"rfc{unused_rfc_number}.txt" + file_in_the_way.touch() + r = self.client.post( + url, data=post_data, format="json", headers={"X-Api-Key": "valid-token"} + ) + self.assertEqual(r.status_code, 409) # conflict + file_in_the_way.unlink() + + # Put a blob in the way. Post should fail because replace = False + blob_in_the_way = Blob.objects.create( + bucket="rfc", name=f"txt/rfc{unused_rfc_number}.txt", content=b"" + ) + r = self.client.post( + url, data=post_data, format="json", headers={"X-Api-Key": "valid-token"} + ) + self.assertEqual(r.status_code, 409) # conflict + blob_in_the_way.delete() + + r = self.client.post( + url, data=post_data, format="json", headers={"X-Api-Key": "valid-token"} + ) + self.assertEqual(r.status_code, 200) + rfc = Document.objects.filter(rfc_number=unused_rfc_number).first() + self.assertIsNotNone(rfc) + self.assertEqual(rfc.came_from_draft(), draft) + self.assertEqual( + rfc.docevent_set.filter( + type="published_rfc", time="2025-12-17T20:29:00Z" + ).count(), + 1, + ) + self.assertEqual(rfc.title, "RFC " + draft.title) + self.assertEqual(rfc.documentauthor_set.count(), 0) + self.assertEqual( + [ + { + "titlepage_name": ra.titlepage_name, + "is_editor": ra.is_editor, + "person": ra.person, + "email": ra.email, + "affiliation": ra.affiliation, + "country": ra.country, + } + for ra in rfc.rfcauthor_set.all() + ], + [ + { + "titlepage_name": f"titlepage {author.name}", + "is_editor": False, + "person": author, + "email": author.email(), + "affiliation": "Some Affiliation", + "country": "CA", + } + for author in rfc_authors + ], + ) + self.assertEqual(rfc.group, rfc_group) + self.assertEqual(rfc.stream_id, rfc_stream_id) + self.assertEqual(rfc.abstract, "RFC version of " + draft.abstract) + self.assertEqual(rfc.pages, draft.pages + 10) + self.assertEqual(rfc.std_level_id, "ps") + self.assertEqual(rfc.ad, rfc_ad) + self.assertEqual(set(rfc.related_that_doc("obs")), set([o for o in obsoletes])) + self.assertEqual( + set(rfc.related_that_doc("updates")), set([o for o in updates]) + ) + self.assertEqual(rfc.part_of(), []) + self.assertEqual(draft.get_state().slug, "rfc") + # todo test non-empty relationships + # todo test references (when updating that is part of the handling) + + self.assertTrue(mock_task_delay.called) + mock_args, mock_kwargs = mock_task_delay.call_args + self.assertIn("rfc_number_list", mock_kwargs) + expected_rfc_number_list = [rfc.rfc_number] + expected_rfc_number_list.extend([d.rfc_number for d in updates + obsoletes]) + expected_rfc_number_list = sorted(set(expected_rfc_number_list)) + self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list) + + @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + @mock.patch("ietf.api.views_rpc.rebuild_reference_relations_task") + @mock.patch("ietf.api.views_rpc.update_rfc_searchindex_task") + @mock.patch("ietf.api.views_rpc.trigger_red_precomputer_task") + def test_upload_rfc_files( + self, + mock_trigger_red_task, + mock_update_searchindex_task, + mock_rebuild_relations, + ): + def _valid_post_data(): + """Generate a valid post data dict + + Each API call needs a fresh set of files, so don't reuse the return + value from this for multiple calls! + """ + return { + "rfc": rfc.rfc_number, + "contents": [ + ContentFile(b"This is .xml", "myfile.xml"), + ContentFile(b"This is .txt", "myfile.txt"), + ContentFile(b"This is .html", "myfile.html"), + ContentFile(b"This is .pdf", "myfile.pdf"), + ContentFile(b"This is .json", "myfile.json"), + ContentFile(b"This is .notprepped.xml", "myfile.notprepped.xml"), + ], + "replace": False, + } + + url = urlreverse("ietf.api.purple_api.upload_rfc_files") + updates = RfcFactory.create_batch(2) + obsoletes = RfcFactory.create_batch(2) + + rfc = WgRfcFactory() + for r in obsoletes: + rfc.relateddocument_set.create(relationship_id="obs", target=r) + for r in updates: + rfc.relateddocument_set.create(relationship_id="updates", target=r) + assert isinstance(rfc, Document), "WgRfcFactory should generate a Document" + rfc_path = Path(settings.RFC_PATH) + (rfc_path / "prerelease").mkdir() + content = StringIO("XML content\n") + content.name = "myrfc.xml" + + # no api key + r = self.client.post(url, _valid_post_data(), format="multipart") + self.assertEqual(r.status_code, 403) + self.assertFalse(mock_update_searchindex_task.delay.called) + + # invalid RFC + r = self.client.post( + url, + _valid_post_data() | {"rfc": rfc.rfc_number + 10}, + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 400) + self.assertFalse(mock_update_searchindex_task.delay.called) + + # empty files + r = self.client.post( + url, + _valid_post_data() + | { + "contents": [ + ContentFile(b"", "myfile.xml"), + ContentFile(b"", "myfile.txt"), + ContentFile(b"", "myfile.html"), + ContentFile(b"", "myfile.pdf"), + ContentFile(b"", "myfile.json"), + ContentFile(b"", "myfile.notprepped.xml"), + ] + }, + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 400) + self.assertFalse(mock_update_searchindex_task.delay.called) + + # bad file type + r = self.client.post( + url, + _valid_post_data() + | { + "contents": [ + ContentFile(b"Some content", "myfile.jpg"), + ] + }, + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 400) + self.assertFalse(mock_update_searchindex_task.delay.called) + + # Put a file in the way. Post should fail because replace = False + file_in_the_way = rfc_path / f"{rfc.name}.txt" + file_in_the_way.touch() + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 409) # conflict + self.assertFalse(mock_update_searchindex_task.delay.called) + file_in_the_way.unlink() + + # Put a blob in the way. Post should fail because replace = False + blob_in_the_way = Blob.objects.create( + bucket="rfc", name=f"txt/{rfc.name}.txt", content=b"" + ) + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 409) # conflict + self.assertFalse(mock_update_searchindex_task.delay.called) + blob_in_the_way.delete() + + # valid post + mock_trigger_red_task.delay.reset_mock() + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 200) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) + for extension in ["xml", "txt", "html", "pdf", "json"]: + filename = f"{rfc.name}.{extension}" + self.assertEqual( + (rfc_path / filename).read_text(), + f"This is .{extension}", + f"{extension} file should contain the expected content", + ) + self.assertEqual( + bytes( + Blob.objects.get( + bucket="rfc", name=f"{extension}/{filename}" + ).content + ), + f"This is .{extension}".encode("utf-8"), + f"{extension} blob should contain the expected content", + ) + # special case for notprepped + notprepped_fn = f"{rfc.name}.notprepped.xml" + self.assertEqual( + (rfc_path / "prerelease" / notprepped_fn).read_text(), + "This is .notprepped.xml", + ".notprepped.xml file should contain the expected content", + ) + self.assertEqual( + bytes( + Blob.objects.get( + bucket="rfc", name=f"notprepped/{notprepped_fn}" + ).content + ), + b"This is .notprepped.xml", + ".notprepped.xml blob should contain the expected content", + ) + # Confirm that the red precomputer was triggered correctly + self.assertTrue(mock_trigger_red_task.delay.called) + _, mock_kwargs = mock_trigger_red_task.delay.call_args + self.assertIn("rfc_number_list", mock_kwargs) + expected_rfc_number_list = [rfc.rfc_number] + expected_rfc_number_list.extend([d.rfc_number for d in updates + obsoletes]) + expected_rfc_number_list = sorted(set(expected_rfc_number_list)) + self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list) + # Confirm that the search index update task was called correctly + self.assertTrue(mock_update_searchindex_task.delay.called) + # Confirm reference relations rebuild task was called correctly + self.assertTrue(mock_rebuild_relations.delay.called) + _, mock_kwargs = mock_rebuild_relations.delay.call_args + self.assertIn("doc_names", mock_kwargs) + self.assertEqual(mock_kwargs["doc_names"], [rfc.name]) + + # re-post with replace = False should now fail + mock_update_searchindex_task.reset_mock() + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 409) # conflict + self.assertFalse(mock_update_searchindex_task.delay.called) + + # re-post with replace = True should succeed + r = self.client.post( + url, + _valid_post_data() | {"replace": True}, + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 200) + self.assertTrue(mock_update_searchindex_task.delay.called) + self.assertEqual( + mock_update_searchindex_task.delay.call_args, + mock.call(rfc.rfc_number), + ) + + @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + def test_refresh_rfc_index(self): + DirtyBits.objects.create( + slug=DirtyBits.Slugs.RFCINDEX, + dirty_time=timezone.now() - datetime.timedelta(days=1), + processed_time=timezone.now() - datetime.timedelta(hours=12), + ) + self.assertFalse(rfcindex_is_dirty()) + url = urlreverse("ietf.api.purple_api.refresh_rfc_index") + response = self.client.get(url) + self.assertEqual(response.status_code, 403) + response = self.client.get(url, headers={"X-Api-Key": "invalid-token"}) + self.assertEqual(response.status_code, 403) + response = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(response.status_code, 405) + self.assertFalse(rfcindex_is_dirty()) + response = self.client.post(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(response.status_code, 202) + self.assertTrue(rfcindex_is_dirty()) + + def test_destination_helper_mixin_fs_destination(self): + file_list = [f"rfc31337.{ext}" for ext in ["txt", "xml", "pdf", "html"]] + for filename in file_list: + self.assertEqual( + DestinationHelperMixin().fs_destination(filename), + Path(f"{settings.RFC_PATH}") / filename, + ) + # noteprepped xml + filename = "rfc31337.notprepped.xml" + self.assertEqual( + DestinationHelperMixin().fs_destination(filename), + Path(f"{settings.RFC_PATH}/prerelease") / filename, + ) + + def test_destination_helper_mixin_blob_destination(self): + file_list = {ext: f"rfc31337.{ext}" for ext in ["txt", "xml", "pdf", "html"]} + for file_type, filename in file_list.items(): + self.assertEqual( + DestinationHelperMixin().blob_destination(filename), + f"{file_type}/{filename}", + ) + # noteprepped xml + filename = "rfc31337.notprepped.xml" + self.assertEqual( + DestinationHelperMixin().blob_destination(filename), + f"notprepped/{filename}", + ) + + @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + @mock.patch("ietf.api.views_rpc.process_rpc_queue_task.delay") + def test_process_rpc_queue(self, mock_task_delay): + url = urlreverse("ietf.api.purple_api.process_rpc_queue") + queue_entries = [ + { + "id": 9850, + "name": "draft-ietf-netmod-system-config", + "title": "System-defined Configuration", + "draft_url": "http://localhost:8000/doc/draft-ietf-netmod-system-config-20", + "disposition": "in_progress", + "external_deadline": None, + "labels": [], + "cluster": None, + "assignment_set": [ + { + "id": 434, + "rfc_to_be": 9850, + "role": "first_editor", + "state": "in_progress", + } + ], + "actionholder_set": [], + "pending_activities": [], + "rfc_number": None, + "pages": 33, + "enqueued_at": "2026-01-26T12:00:00Z", + "final_approval": [], + "iana_status": { + "slug": "completed", + "name": "completed", + "desc": "IANA has completed actions in draft", + }, + "blocking_reasons": [], + "authors": [{"titlepage_name": "Q. Ma", "is_editor": True}], + "approval_log_message": [], + "stream": "ietf", + "group": "netmod", + "group_name": "Network Modeling", + "std_level": "ps", + "references": [], + "rev": "20", + } + ] + queue_data = {"data": queue_entries} + + # no credentials + response = self.client.post( + url, data=queue_data, content_type="application/json" + ) + self.assertEqual(response.status_code, 403) + mock_task_delay.assert_not_called() + + # invalid token + response = self.client.post( + url, + data=queue_data, + content_type="application/json", + headers={"X-Api-Key": "invalid-token"}, + ) + self.assertEqual(response.status_code, 403) + mock_task_delay.assert_not_called() + + # valid token, wrong method + response = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(response.status_code, 405) + mock_task_delay.assert_not_called() + + # valid token, missing "data" field + response = self.client.post( + url, + data={}, + content_type="application/json", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(response.status_code, 400) + mock_task_delay.assert_not_called() + + # valid token, POST with data + response = self.client.post( + url, + data=queue_data, + content_type="application/json", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(response.status_code, 202) + mock_task_delay.assert_called_once_with(queue_entries) diff --git a/ietf/api/urls.py b/ietf/api/urls.py index b0dbaf91ce..7a082567b8 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -1,26 +1,31 @@ # Copyright The IETF Trust 2017-2024, All Rights Reserved +from drf_spectacular.views import SpectacularAPIView + from django.conf import settings -from django.urls import include +from django.urls import include, path from django.views.generic import TemplateView from ietf import api -from ietf.doc import views_ballot +from ietf.doc import views_ballot, api as doc_api from ietf.meeting import views as meeting_views from ietf.submit import views as submit_views from ietf.utils.urls import url from . import views as api_views +from .routers import PrefixedSimpleRouter # DRF API routing - disabled until we plan to use it -# from drf_spectacular.views import SpectacularAPIView -# from django.urls import path # from ietf.person import api as person_api -# from .routers import PrefixedSimpleRouter # core_router = PrefixedSimpleRouter(name_prefix="ietf.api.core_api") # core api router # core_router.register("email", person_api.EmailViewSet) # core_router.register("person", person_api.PersonViewSet) +# todo more general name for this API? +red_router = PrefixedSimpleRouter(name_prefix="ietf.api.red_api") # red api router +red_router.register("doc", doc_api.RfcViewSet) +red_router.register("subseries", doc_api.SubseriesViewSet, basename="subseries") + api.autodiscover() urlpatterns = [ @@ -32,7 +37,9 @@ url(r'^v2/person/person', api_views.ApiV2PersonExportView.as_view()), # --- DRF API --- # path("core/", include(core_router.urls)), - # path("schema/", SpectacularAPIView.as_view()), + path("purple/", include("ietf.api.urls_rpc")), + path("red/", include(red_router.urls)), + path("schema/", SpectacularAPIView.as_view()), # # --- Custom API endpoints, sorted alphabetically --- # Email alias information for drafts @@ -49,6 +56,9 @@ url(r'^group/role-holder-addresses/$', api_views.role_holder_addresses), # Let IESG members set positions programmatically url(r'^iesg/position', views_ballot.api_set_position), + # Find the blob to store for a given materials document path + url(r'^meeting/(?:(?P(?:interim-)?[a-z0-9-]+)/)?materials/%(document)s(?P\.[A-Za-z0-9]+)?/resolve-cached/$' % settings.URL_REGEXPS, meeting_views.api_resolve_materials_name_cached), + url(r'^meeting/blob/(?P[a-z0-9-]+)/(?P[a-z][a-z0-9.-]+)$', meeting_views.api_retrieve_materials_blob), # Let Meetecho set session video URLs url(r'^meeting/session/video/url$', meeting_views.api_set_session_video_url), # Let Meetecho tell us the name of its recordings @@ -66,12 +76,14 @@ # Let MeetEcho upload session polls url(r'^notify/session/polls/?$', meeting_views.api_upload_polls), # Let the registration system notify us about registrations - url(r'^notify/meeting/registration/?', api_views.api_new_meeting_registration), + url(r'^notify/meeting/registration/v2/?', api_views.api_new_meeting_registration_v2), # OpenID authentication provider url(r'^openid/$', TemplateView.as_view(template_name='api/openid-issuer.html'), name='ietf.api.urls.oidc_issuer'), url(r'^openid/', include('oidc_provider.urls', namespace='oidc_provider')), # Email alias listing url(r'^person/email/$', api_views.active_email_list), + # Related Email listing + url(r'^person/email/(?P[^/\x00]+)/related/$', api_views.related_email_list), # Draft submission API url(r'^submit/?$', submit_views.api_submit_tombstone), # Draft upload API diff --git a/ietf/api/urls_rpc.py b/ietf/api/urls_rpc.py new file mode 100644 index 0000000000..07f2cf8751 --- /dev/null +++ b/ietf/api/urls_rpc.py @@ -0,0 +1,52 @@ +# Copyright The IETF Trust 2023-2026, All Rights Reserved +from django.urls import include, path + +from ietf.api import views_rpc +from ietf.api.routers import PrefixedDefaultRouter +from ietf.utils.urls import url + +router = PrefixedDefaultRouter(use_regex_path=False, name_prefix="ietf.api.purple_api") +router.include_format_suffixes = False +router.register(r"draft", views_rpc.DraftViewSet, basename="draft") +router.register(r"person", views_rpc.PersonViewSet) +router.register(r"rfc", views_rpc.RfcViewSet, basename="rfc") + +router.register( + r"rfc//authors", + views_rpc.RfcAuthorViewSet, + basename="rfc-authors", +) + +urlpatterns = [ + url(r"^doc/drafts_by_names/", views_rpc.DraftsByNamesView.as_view()), + url(r"^persons/search/", views_rpc.RpcPersonSearch.as_view()), + path( + r"rfc/publish/", + views_rpc.RfcPubNotificationView.as_view(), + name="ietf.api.purple_api.notify_rfc_published", + ), + path( + r"rfc/publish/files/", + views_rpc.RfcPubFilesView.as_view(), + name="ietf.api.purple_api.upload_rfc_files", + ), + path( + r"rfc_index/refresh/", + views_rpc.RfcIndexView.as_view(), + name="ietf.api.purple_api.refresh_rfc_index", + ), + path(r"subject//person/", views_rpc.SubjectPersonView.as_view()), + path( + r"queue/process/", + views_rpc.ProcessRpcQueueView.as_view(), + name="ietf.api.purple_api.process_rpc_queue", + ), +] + +# add routers at the end so individual routes can steal parts of their address +# space (e.g., ^rfc/publish/ superseding the ^rfc/ routes of RfcViewSet) +urlpatterns.extend( + [ + path("", include(router.urls)), + ] +) diff --git a/ietf/api/views.py b/ietf/api/views.py index 2fd9d2730f..420bc39693 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -16,9 +16,7 @@ from django.contrib.auth import authenticate from django.contrib.auth.decorators import login_required from django.contrib.auth.models import User -from django.core.exceptions import ValidationError -from django.core.validators import validate_email -from django.http import HttpResponse, Http404, JsonResponse +from django.http import HttpResponse, Http404, JsonResponse, HttpResponseBadRequest from django.shortcuts import render, get_object_or_404 from django.urls import reverse from django.utils.decorators import method_decorator @@ -43,13 +41,11 @@ from ietf.doc.utils import DraftAliasGenerator, fuzzy_find_documents from ietf.group.utils import GroupAliasGenerator, role_holder_emails from ietf.ietfauth.utils import role_required -from ietf.ietfauth.views import send_account_creation_email from ietf.ipr.utils import ingest_response_email as ipr_ingest_response_email from ietf.meeting.models import Meeting -from ietf.nomcom.models import Volunteer, NomCom +from ietf.meeting.utils import import_registration_json_validator, process_single_registration from ietf.nomcom.utils import ingest_feedback_email as nomcom_ingest_feedback_email from ietf.person.models import Person, Email -from ietf.stats.models import MeetingRegistration from ietf.sync.iana import ingest_review_email as iana_ingest_review_email from ietf.utils import log from ietf.utils.decorators import require_api_key @@ -68,7 +64,10 @@ def top_level(request): } serializer = Serializer() - desired_format = determine_format(request, serializer) + try: + desired_format = determine_format(request, serializer) + except BadRequest as err: + return HttpResponseBadRequest(str(err)) options = {} @@ -76,10 +75,12 @@ def top_level(request): callback = request.GET.get('callback', 'callback') if not is_valid_jsonp_callback_value(callback): - raise BadRequest('JSONP callback name is invalid.') + return HttpResponseBadRequest("JSONP callback name is invalid") options['callback'] = callback + # This might raise UnsupportedFormat, but that indicates a real server misconfiguration + # so let it bubble up unhandled and trigger a 500 / email to admins. serialized = serializer.serialize(available_resources, desired_format, options) return HttpResponse(content=serialized, content_type=build_content_type(desired_format)) @@ -96,7 +97,7 @@ class PersonalInformationExportView(DetailView, JsonExportMixin): def get(self, request): person = get_object_or_404(self.model, user=request.user) - expand = ['searchrule', 'documentauthor', 'ad_document_set', 'ad_dochistory_set', 'docevent', + expand = ['searchrule', 'documentauthor', 'rfcauthor', 'ad_document_set', 'ad_dochistory_set', 'docevent', 'ballotpositiondocevent', 'deletedevent', 'email_set', 'groupevent', 'role', 'rolehistory', 'iprdisclosurebase', 'iprevent', 'liaisonstatementevent', 'allowlisted', 'schedule', 'constraint', 'schedulingevent', 'message', 'sendqueue', 'nominee', 'topicfeedbacklastseen', 'alias', 'email', 'apikeys', 'personevent', @@ -113,7 +114,11 @@ class ApiV2PersonExportView(DetailView, JsonExportMixin): model = Person def err(self, code, text): - return HttpResponse(text, status=code, content_type='text/plain') + return HttpResponse( + text, + status=code, + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + ) def post(self, request): querydict = request.POST.copy() @@ -145,95 +150,61 @@ def post(self, request): # else: # return HttpResponse(status=405) -@require_api_key -@role_required('Robot') + +@requires_api_token @csrf_exempt -def api_new_meeting_registration(request): +def api_new_meeting_registration_v2(request): '''REST API to notify the datatracker about a new meeting registration''' - def err(code, text): - return HttpResponse(text, status=code, content_type='text/plain') - required_fields = [ 'meeting', 'first_name', 'last_name', 'affiliation', 'country_code', - 'email', 'reg_type', 'ticket_type', 'checkedin', 'is_nomcom_volunteer'] - fields = required_fields + [] - if request.method == 'POST': - # parameters: - # apikey: - # meeting - # name - # email - # reg_type (In Person, Remote, Hackathon Only) - # ticket_type (full_week, one_day, student) - # - data = {'attended': False, } - missing_fields = [] - for item in fields: - value = request.POST.get(item, None) - if value is None and item in required_fields: - missing_fields.append(item) - data[item] = value - if missing_fields: - return err(400, "Missing parameters: %s" % ', '.join(missing_fields)) - number = data['meeting'] - try: - meeting = Meeting.objects.get(number=number) - except Meeting.DoesNotExist: - return err(400, "Invalid meeting value: '%s'" % (number, )) - reg_type = data['reg_type'] - email = data['email'] - try: - validate_email(email) - except ValidationError: - return err(400, "Invalid email value: '%s'" % (email, )) - if request.POST.get('cancelled', 'false') == 'true': - MeetingRegistration.objects.filter( - meeting_id=meeting.pk, - email=email, - reg_type=reg_type).delete() - return HttpResponse('OK', status=200, content_type='text/plain') - else: - object, created = MeetingRegistration.objects.get_or_create( - meeting_id=meeting.pk, - email=email, - reg_type=reg_type) - try: - # Update attributes - for key in set(data.keys())-set(['attended', 'apikey', 'meeting', 'email']): - if key == 'checkedin': - new = bool(data.get(key).lower() == 'true') - else: - new = data.get(key) - setattr(object, key, new) - person = Person.objects.filter(email__address=email) - if person.exists(): - object.person = person.first() - object.save() - except ValueError as e: - return err(400, "Unexpected POST data: %s" % e) - response = "Accepted, New registration" if created else "Accepted, Updated registration" - if User.objects.filter(username__iexact=email).exists() or Email.objects.filter(address=email).exists(): - pass - else: - send_account_creation_email(request, email) - response += ", Email sent" - - # handle nomcom volunteer - if request.POST.get('is_nomcom_volunteer', 'false').lower() == 'true' and object.person: - try: - nomcom = NomCom.objects.get(is_accepting_volunteers=True) - except (NomCom.DoesNotExist, NomCom.MultipleObjectsReturned): - nomcom = None - if nomcom: - Volunteer.objects.get_or_create( - nomcom=nomcom, - person=object.person, - defaults={ - "affiliation": data["affiliation"], - "origin": "registration" - } - ) - return HttpResponse(response, status=202, content_type='text/plain') - else: - return HttpResponse(status=405) + def _http_err(code, text): + return HttpResponse( + text, + status=code, + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + ) + + def _api_response(result): + return JsonResponse(data={"result": result}) + + if request.method != "POST": + return _http_err(405, "Method not allowed") + + if request.content_type != "application/json": + return _http_err(415, "Content-Type must be application/json") + + # Validate + try: + payload = json.loads(request.body) + import_registration_json_validator.validate(payload) + except json.decoder.JSONDecodeError as err: + return _http_err(400, f"JSON parse error at line {err.lineno} col {err.colno}: {err.msg}") + except jsonschema.exceptions.ValidationError as err: + return _http_err(400, f"JSON schema error at {err.json_path}: {err.message}") + except Exception: + return _http_err(400, "Invalid request format") + + # Get the meeting ID from the first registration, the API only deals with one meeting at a time + first_email = next(iter(payload['objects'])) + meeting_number = payload['objects'][first_email]['meeting'] + try: + meeting = Meeting.objects.get(number=meeting_number) + except Meeting.DoesNotExist: + return _http_err(400, f"Invalid meeting value: {meeting_number}") + + # confirm email exists + try: + Email.objects.get(address=first_email) + except Email.DoesNotExist: + return _http_err(400, f"Unknown email: {first_email}") + + reg_data = payload['objects'][first_email] + + process_single_registration(reg_data, meeting) + + return HttpResponse( + 'Success', + status=202, + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + ) def version(request): @@ -545,6 +516,35 @@ def active_email_list(request): return HttpResponse(status=405) +@requires_api_token +@csrf_exempt +def related_email_list(request, email): + """Given an email address, returns all other email addresses known + to Datatracker, via Person object + """ + def _http_err(code, text): + return HttpResponse( + text, + status=code, + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + ) + + if request.method == "GET": + try: + email_obj = Email.objects.get(address=email) + except Email.DoesNotExist: + return _http_err(404, "Email not found") + person = email_obj.person + if not person: + return JsonResponse({"addresses": []}) + return JsonResponse( + { + "addresses": list(person.email_set.values_list("address", flat=True)), + } + ) + return HttpResponse(status=405) + + @requires_api_token def role_holder_addresses(request): if request.method == "GET": @@ -653,7 +653,11 @@ def ingest_email_handler(request, test_mode=False): """ def _http_err(code, text): - return HttpResponse(text, status=code, content_type="text/plain") + return HttpResponse( + text, + status=code, + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + ) def _api_response(result): return JsonResponse(data={"result": result}) diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py new file mode 100644 index 0000000000..83d0abefb1 --- /dev/null +++ b/ietf/api/views_rpc.py @@ -0,0 +1,600 @@ +# Copyright The IETF Trust 2023-2026, All Rights Reserved +import os +import shutil +from pathlib import Path +from tempfile import TemporaryDirectory + +from django.conf import settings +from django.db import IntegrityError +from drf_spectacular.utils import OpenApiParameter +from rest_framework import mixins, parsers, serializers, viewsets, status +from rest_framework.decorators import action +from rest_framework.exceptions import APIException +from rest_framework.views import APIView +from rest_framework.response import Response + +from django.db.models import CharField as ModelCharField, OuterRef, Subquery, Q +from django.db.models.functions import Coalesce +from django.http import Http404 +from drf_spectacular.utils import extend_schema_view, extend_schema +from rest_framework import generics +from rest_framework.fields import CharField as DrfCharField +from rest_framework.filters import SearchFilter +from rest_framework.pagination import LimitOffsetPagination + +from ietf.api.serializers_rpc import ( + PersonSerializer, + FullDraftSerializer, + DraftSerializer, + SubmittedToQueueSerializer, + OriginalStreamSerializer, + ReferenceSerializer, + EmailPersonSerializer, + RfcWithAuthorsSerializer, + DraftWithAuthorsSerializer, + NotificationAckSerializer, + RfcPubSerializer, + RfcFileSerializer, + EditableRfcSerializer, +) +from ietf.doc.models import Document, DocHistory, RfcAuthor, DocEvent +from ietf.doc.serializers import RfcAuthorSerializer +from ietf.doc.storage_utils import remove_from_storage, store_file, exists_in_storage +from ietf.doc.tasks import ( + signal_update_rfc_metadata_task, + rebuild_reference_relations_task, + trigger_red_precomputer_task, + update_rfc_searchindex_task, +) +from ietf.person.models import Email, Person +from ietf.sync.rfcindex import mark_rfcindex_as_dirty +from ietf.sync.tasks import process_rpc_queue_task + + +class Conflict(APIException): + status_code = status.HTTP_409_CONFLICT + default_detail = "Conflict." + default_code = "conflict" + + +@extend_schema_view( + retrieve=extend_schema( + operation_id="get_person_by_id", + summary="Find person by ID", + description="Returns a single person", + parameters=[ + OpenApiParameter( + name="person_id", + type=int, + location="path", + description="Person ID identifying this person.", + ), + ], + ), +) +class PersonViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet): + queryset = Person.objects.all() + serializer_class = PersonSerializer + api_key_endpoint = "ietf.api.views_rpc" + lookup_url_kwarg = "person_id" + + @extend_schema( + operation_id="get_persons", + summary="Get a batch of persons", + description="Returns a list of persons matching requested ids. Omits any that are missing.", + request=list[int], + responses=PersonSerializer(many=True), + ) + @action(detail=False, methods=["post"]) + def batch(self, request): + """Get a batch of rpc person names""" + pks = request.data + return Response( + self.get_serializer(Person.objects.filter(pk__in=pks), many=True).data + ) + + @extend_schema( + operation_id="persons_by_email", + summary="Get a batch of persons by email addresses", + description=( + "Returns a list of persons matching requested ids. " + "Omits any that are missing." + ), + request=list[str], + responses=EmailPersonSerializer(many=True), + ) + @action(detail=False, methods=["post"], serializer_class=EmailPersonSerializer) + def batch_by_email(self, request): + emails = Email.objects.filter(address__in=request.data, person__isnull=False) + serializer = self.get_serializer(emails, many=True) + return Response(serializer.data) + + +class SubjectPersonView(APIView): + api_key_endpoint = "ietf.api.views_rpc" + + @extend_schema( + operation_id="get_subject_person_by_id", + summary="Find person for OIDC subject by ID", + description="Returns a single person", + responses=PersonSerializer, + parameters=[ + OpenApiParameter( + name="subject_id", + type=str, + description="subject ID of person to return", + location="path", + ), + ], + ) + def get(self, request, subject_id: str): + try: + user_id = int(subject_id) + except ValueError: + raise serializers.ValidationError( + {"subject_id": "This field must be an integer value."} + ) + person = Person.objects.filter(user__pk=user_id).first() + if person: + return Response(PersonSerializer(person).data) + raise Http404 + + +class RpcLimitOffsetPagination(LimitOffsetPagination): + default_limit = 10 + max_limit = 100 + + +class SingleTermSearchFilter(SearchFilter): + """SearchFilter backend that does not split terms + + The default SearchFilter treats comma or whitespace-separated terms as individual + search terms. This backend instead searches for the exact term. + """ + + def get_search_terms(self, request): + value = request.query_params.get(self.search_param, "") + field = DrfCharField(trim_whitespace=False, allow_blank=True) + cleaned_value = field.run_validation(value) + return [cleaned_value] + + +@extend_schema_view( + get=extend_schema( + operation_id="search_person", + description="Get a list of persons, matching by partial name or email", + ), +) +class RpcPersonSearch(generics.ListAPIView): + # n.b. the OpenAPI schema for this can be generated by running + # ietf/manage.py spectacular --file spectacular.yaml + # and extracting / touching up the rpc_person_search_list operation + api_key_endpoint = "ietf.api.views_rpc" + queryset = Person.objects.all() + serializer_class = PersonSerializer + pagination_class = RpcLimitOffsetPagination + + # Searchable on all name-like fields or email addresses + filter_backends = [SingleTermSearchFilter] + search_fields = ["name", "plain", "email__address"] + + +@extend_schema_view( + retrieve=extend_schema( + operation_id="get_draft_by_id", + summary="Get a draft", + description="Returns the draft for the requested ID", + parameters=[ + OpenApiParameter( + name="doc_id", + type=int, + location="path", + description="Doc ID identifying this draft.", + ), + ], + ), + submitted_to_rpc=extend_schema( + operation_id="submitted_to_rpc", + summary="List documents ready to enter the RFC Editor Queue", + description="List documents ready to enter the RFC Editor Queue", + responses=SubmittedToQueueSerializer(many=True), + ), +) +class DraftViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet): + queryset = Document.objects.filter(type_id="draft") + serializer_class = FullDraftSerializer + api_key_endpoint = "ietf.api.views_rpc" + lookup_url_kwarg = "doc_id" + + @action(detail=False, serializer_class=SubmittedToQueueSerializer) + def submitted_to_rpc(self, request): + """Return documents in datatracker that have been submitted to the RPC but are not yet in the queue + + Those queries overreturn - there may be things, particularly not from the IETF stream that are already in the queue. + """ + ietf_docs = Q(states__type_id="draft-iesg", states__slug__in=["ann"]) + irtf_iab_ise_editorial_docs = Q( + states__type_id__in=[ + "draft-stream-iab", + "draft-stream-irtf", + "draft-stream-ise", + "draft-stream-editorial", + ], + states__slug__in=["rfc-edit"], + ) + docs = ( + self.get_queryset() + .filter(type_id="draft") + .filter(ietf_docs | irtf_iab_ise_editorial_docs) + ) + serializer = self.get_serializer(docs, many=True) + return Response(serializer.data) + + @extend_schema( + operation_id="get_draft_references", + summary="Get normative references to I-Ds", + description=( + "Returns the id and name of each normatively " + "referenced Internet-Draft for the given docId" + ), + parameters=[ + OpenApiParameter( + name="doc_id", + type=int, + location="path", + description="Doc ID identifying this draft.", + ), + ], + responses=ReferenceSerializer(many=True), + ) + @action(detail=True, serializer_class=ReferenceSerializer) + def references(self, request, doc_id=None): + doc = self.get_object() + serializer = self.get_serializer( + [ + reference + for reference in doc.related_that_doc("refnorm") + if reference.type_id == "draft" + ], + many=True, + ) + return Response(serializer.data) + + @extend_schema( + operation_id="get_draft_authors", + summary="Gather authors of the drafts with the given names", + description="returns a list mapping draft names to objects describing authors", + request=list[str], + responses=DraftWithAuthorsSerializer(many=True), + ) + @action(detail=False, methods=["post"], serializer_class=DraftWithAuthorsSerializer) + def bulk_authors(self, request): + drafts = self.get_queryset().filter(name__in=request.data) + serializer = self.get_serializer(drafts, many=True) + return Response(serializer.data) + + +@extend_schema_view( + rfc_original_stream=extend_schema( + operation_id="get_rfc_original_streams", + summary="Get the streams RFCs were originally published into", + description="returns a list of dicts associating an RFC with its originally published stream", + responses=OriginalStreamSerializer(many=True), + ) +) +class RfcViewSet(mixins.UpdateModelMixin, viewsets.GenericViewSet): + queryset = Document.objects.filter(type_id="rfc") + api_key_endpoint = "ietf.api.views_rpc" + lookup_field = "rfc_number" + serializer_class = EditableRfcSerializer + + def perform_update(self, serializer): + DocEvent.objects.create( + doc=serializer.instance, + rev=serializer.instance.rev, + by=Person.objects.get(name="(System)"), + type="sync_from_rfc_editor", + desc="Metadata update from RFC Editor", + ) + super().perform_update(serializer) + + @action(detail=False, serializer_class=OriginalStreamSerializer) + def rfc_original_stream(self, request): + rfcs = self.get_queryset().annotate( + orig_stream_id=Coalesce( + Subquery( + DocHistory.objects.filter(doc=OuterRef("pk")) + .exclude(stream__isnull=True) + .order_by("time") + .values_list("stream_id", flat=True)[:1] + ), + "stream_id", + output_field=ModelCharField(), + ), + ) + serializer = self.get_serializer(rfcs, many=True) + return Response(serializer.data) + + @extend_schema( + operation_id="get_rfc_authors", + summary="Gather authors of the RFCs with the given numbers", + description="returns a list mapping rfc numbers to objects describing authors", + request=list[int], + responses=RfcWithAuthorsSerializer(many=True), + ) + @action(detail=False, methods=["post"], serializer_class=RfcWithAuthorsSerializer) + def bulk_authors(self, request): + rfcs = self.get_queryset().filter(rfc_number__in=request.data) + serializer = self.get_serializer(rfcs, many=True) + return Response(serializer.data) + + +class DraftsByNamesView(APIView): + api_key_endpoint = "ietf.api.views_rpc" + + @extend_schema( + operation_id="get_drafts_by_names", + summary="Get a batch of drafts by draft names", + description="returns a list of drafts with matching names", + request=list[str], + responses=DraftSerializer(many=True), + ) + def post(self, request): + names = request.data + docs = Document.objects.filter(type_id="draft", name__in=names) + return Response(DraftSerializer(docs, many=True).data) + + +class RfcAuthorViewSet(viewsets.ReadOnlyModelViewSet): + """ViewSet for RfcAuthor model + + Router needs to provide rfc_number as a kwarg + """ + + api_key_endpoint = "ietf.api.views_rpc" + + queryset = RfcAuthor.objects.all() + serializer_class = RfcAuthorSerializer + lookup_url_kwarg = "author_id" + rfc_number_param = "rfc_number" + + def get_queryset(self): + return ( + super() + .get_queryset() + .filter( + document__type_id="rfc", + document__rfc_number=self.kwargs[self.rfc_number_param], + ) + ) + + +class DestinationHelperMixin: + def fs_destination(self, filename: str | Path) -> Path: + """Destination for an uploaded RFC file in the filesystem + + Strips any path components in filename and returns an absolute Path. + """ + rfc_path = Path(settings.RFC_PATH) + filename = Path(filename) # could potentially have directory components + extension = "".join(filename.suffixes) + if extension == ".notprepped.xml": + return rfc_path / "prerelease" / filename.name + return rfc_path / filename.name + + def blob_destination(self, filename: str | Path) -> str: + """Destination name for an uploaded RFC file in the blob store + + Strips any path components in filename and returns an absolute Path. + """ + filename = Path(filename) # could potentially have directory components + extension = "".join(filename.suffixes) + if extension == ".notprepped.xml": + file_type = "notprepped" + elif extension[0] == ".": + file_type = extension[1:] + else: + raise serializers.ValidationError( + f"Extension does not begin with '.'!? ({filename})", + ) + return f"{file_type}/{filename.name}" + + +class RfcPubNotificationView(DestinationHelperMixin, APIView): + api_key_endpoint = "ietf.api.views_rpc" + + @extend_schema( + operation_id="notify_rfc_published", + summary="Notify datatracker of RFC publication", + request=RfcPubSerializer, + responses=NotificationAckSerializer, + ) + def post(self, request): + serializer = RfcPubSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + # Check blobstore & filesystem for conflicts + rfc_number = serializer.validated_data["rfc_number"] + dest_stem = f"rfc{rfc_number}" + blob_kind = "rfc" + possible_rfc_files = [ + self.fs_destination(dest_stem + ext) + for ext in RfcFileSerializer.allowed_extensions + ] + possible_rfc_blobs = [ + self.blob_destination(dest_stem + ext) + for ext in RfcFileSerializer.allowed_extensions + ] + for possible_existing_file in possible_rfc_files: + if possible_existing_file.exists(): + raise Conflict( + "File(s) already exist for this RFC", + code="files-exist", + ) + for possible_existing_blob in possible_rfc_blobs: + if exists_in_storage(kind=blob_kind, name=possible_existing_blob): + raise Conflict( + "Blob(s) already exist for this RFC", + code="blobs-exist", + ) + # Create RFC + try: + rfc = serializer.save() + except IntegrityError as err: + if Document.objects.filter( + rfc_number=serializer.validated_data["rfc_number"] + ): + raise serializers.ValidationError( + "RFC with that number already exists", + code="rfc-number-in-use", + ) + raise serializers.ValidationError( + f"Unable to publish: {err}", + code="unknown-integrity-error", + ) + rfc_number_list = [rfc.rfc_number] + rfc_number_list.extend( + [d.rfc_number for d in rfc.related_that_doc(("updates", "obs"))] + ) + rfc_number_list = sorted(set(rfc_number_list)) + signal_update_rfc_metadata_task.delay(rfc_number_list=rfc_number_list) + return Response(NotificationAckSerializer().data) + + +class RfcPubFilesView(DestinationHelperMixin, APIView): + api_key_endpoint = "ietf.api.views_rpc" + parser_classes = [parsers.MultiPartParser] + + @extend_schema( + operation_id="upload_rfc_files", + summary="Upload files for a published RFC", + request=RfcFileSerializer, + responses=NotificationAckSerializer, + ) + def post(self, request): + serializer = RfcFileSerializer( + # many=True, + data=request.data, + ) + serializer.is_valid(raise_exception=True) + rfc = serializer.validated_data["rfc"] + uploaded_files = serializer.validated_data["contents"] # list[UploadedFile] + replace = serializer.validated_data["replace"] + dest_stem = f"rfc{rfc.rfc_number}" + mtime = serializer.validated_data["mtime"] + mtimestamp = mtime.timestamp() + blob_kind = "rfc" + + # List of files that might exist for an RFC + possible_rfc_files = [ + self.fs_destination(dest_stem + ext) + for ext in serializer.allowed_extensions + ] + possible_rfc_blobs = [ + self.blob_destination(dest_stem + ext) + for ext in serializer.allowed_extensions + ] + if not replace: + # this is the default: refuse to overwrite anything if not replacing + for possible_existing_file in possible_rfc_files: + if possible_existing_file.exists(): + raise Conflict( + "File(s) already exist for this RFC", + code="files-exist", + ) + for possible_existing_blob in possible_rfc_blobs: + if exists_in_storage(kind=blob_kind, name=possible_existing_blob): + raise Conflict( + "Blob(s) already exist for this RFC", + code="blobs-exist", + ) + + with TemporaryDirectory() as tempdir: + # Save files in a temporary directory. Use the uploaded filename + # extensions to identify files, but ignore the stems and generate our own. + files_to_move = [] # list[Path] + tmpfile_stem = Path(tempdir) / dest_stem + for upfile in uploaded_files: + uploaded_filename = Path(upfile.name) # name supplied by request + uploaded_ext = "".join(uploaded_filename.suffixes) + tempfile_path = tmpfile_stem.with_suffix(uploaded_ext) + with tempfile_path.open("wb") as dest: + for chunk in upfile.chunks(): + dest.write(chunk) + os.utime(tempfile_path, (mtimestamp, mtimestamp)) + files_to_move.append(tempfile_path) + # copy files to final location, removing any existing ones first if the + # remove flag was set + if replace: + for possible_existing_file in possible_rfc_files: + possible_existing_file.unlink(missing_ok=True) + for possible_existing_blob in possible_rfc_blobs: + remove_from_storage( + blob_kind, possible_existing_blob, warn_if_missing=False + ) + for ftm in files_to_move: + with ftm.open("rb") as f: + store_file( + kind=blob_kind, + name=self.blob_destination(ftm), + file=f, + doc_name=rfc.name, + doc_rev=rfc.rev, # expect blank, but match whatever it is + mtime=mtime, + ) + destination = self.fs_destination(ftm) + if ( + settings.SERVER_MODE != "production" + and not destination.parent.exists() + ): + destination.parent.mkdir() + shutil.move(ftm, destination) + + # Trigger red precomputer + needs_updating = [rfc.rfc_number] + for rel in rfc.relateddocument_set.filter( + relationship_id__in=["obs", "updates"] + ): + needs_updating.append(rel.target.rfc_number) + trigger_red_precomputer_task.delay(rfc_number_list=sorted(needs_updating)) + # Trigger search index update + update_rfc_searchindex_task.delay(rfc.rfc_number) + # Trigger reference relation srebuild + rebuild_reference_relations_task.delay(doc_names=[rfc.name]) + + return Response(NotificationAckSerializer().data) + + +class RfcIndexView(APIView): + api_key_endpoint = "ietf.api.views_rpc" + + @extend_schema( + operation_id="refresh_rfc_index", + summary="Refresh rfc-index files", + description="Requests creation of various index files.", + responses={202: None}, + request=None, + ) + def post(self, request): + mark_rfcindex_as_dirty() + return Response(status=202) + + +class RpcQueueDataSerializer(serializers.Serializer): + data = serializers.JSONField() + + +class ProcessRpcQueueView(APIView): + api_key_endpoint = "ietf.api.views_rpc" + + @extend_schema( + operation_id="process_rpc_queue", + summary="Process the provided RPC queue", + description="Schedules parsing the provided queue to update documents with change dqueue data", + responses={202: None}, + request=RpcQueueDataSerializer, + ) + def post(self, request): + serializer = RpcQueueDataSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + process_rpc_queue_task.delay(serializer.validated_data["data"]) + return Response(status=202) diff --git a/ietf/bin/aliases-from-json.py b/ietf/bin/aliases-from-json.py index a0c383a1ac..0da5d1f8b9 100644 --- a/ietf/bin/aliases-from-json.py +++ b/ietf/bin/aliases-from-json.py @@ -38,7 +38,7 @@ def generate_files(records, adest, vdest, postconfirm, vdomain): vpath = tmppath / "virtual" with apath.open("w") as afile, vpath.open("w") as vfile: - date = datetime.datetime.now(datetime.timezone.utc) + date = datetime.datetime.now(datetime.UTC) signature = f"# Generated by {Path(__file__).absolute()} at {date}\n" afile.write(signature) vfile.write(signature) diff --git a/ietf/secr/sreq/__init__.py b/ietf/blobdb/__init__.py similarity index 100% rename from ietf/secr/sreq/__init__.py rename to ietf/blobdb/__init__.py diff --git a/ietf/blobdb/admin.py b/ietf/blobdb/admin.py new file mode 100644 index 0000000000..44a30d1d7f --- /dev/null +++ b/ietf/blobdb/admin.py @@ -0,0 +1,58 @@ +# Copyright The IETF Trust 2025-2026, All Rights Reserved +from django.contrib import admin +from django.db.models import QuerySet +from django.db.models.functions import Length +from rangefilter.filters import DateRangeQuickSelectListFilterBuilder + +from .apps import get_blobdb +from .models import Blob, ResolvedMaterial +from .utils import queue_for_replication + + +@admin.register(Blob) +class BlobAdmin(admin.ModelAdmin): + list_display = ["bucket", "name", "object_size", "modified", "mtime", "content_type"] + list_filter = [ + "bucket", + "content_type", + ("modified", DateRangeQuickSelectListFilterBuilder()), + ("mtime", DateRangeQuickSelectListFilterBuilder()), + ] + search_fields = ["name"] + list_display_links = ["name"] + actions = ["replicate_blob"] + + def get_queryset(self, request): + return ( + super().get_queryset(request) + .defer("content") # don't load this unless we want it + .annotate(object_size=Length("content")) # accessed via object_size() + ) + + @admin.display(ordering="object_size") + def object_size(self, instance): + """Get the size of the object""" + return instance.object_size # annotation added in get_queryset() + + @admin.action(description="Replicate blobs") + def replicate_blob(self, request, queryset: QuerySet[Blob]): + blob_count = 0 + for blob in queryset.all(): + if isinstance(blob, Blob): + queue_for_replication( + bucket=blob.bucket, name=blob.name, using=get_blobdb() + ) + blob_count += 1 + self.message_user( + request, + f"Queued replication of a total of {blob_count} Blob(s)", + ) + + +@admin.register(ResolvedMaterial) +class ResolvedMaterialAdmin(admin.ModelAdmin): + model = ResolvedMaterial + list_display = ["name", "meeting_number", "bucket", "blob"] + list_filter = ["meeting_number", "bucket"] + search_fields = ["name", "blob"] + ordering = ["name"] diff --git a/ietf/blobdb/apps.py b/ietf/blobdb/apps.py new file mode 100644 index 0000000000..c2513b6819 --- /dev/null +++ b/ietf/blobdb/apps.py @@ -0,0 +1,30 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +from django.apps import AppConfig + + +class BlobdbConfig(AppConfig): + default_auto_field = "django.db.models.BigAutoField" + name = "ietf.blobdb" + + def ready(self): + """Initialize app once the registries / settings are populated""" + from django.conf import settings + + # Validate that the DB is set up + db = get_blobdb() # depends on settings.BLOBDB_DATABASE + if db is not None and db not in settings.DATABASES: + raise RuntimeError( + f"settings.BLOBDB_DATABASE is '{db}' but that is not present in settings.DATABASES" + ) + + # Validate replication settings + from .replication import validate_replication_settings + + validate_replication_settings() + + +def get_blobdb(): + """Retrieve the blobdb setting from Django's settings""" + from django.conf import settings + + return getattr(settings, "BLOBDB_DATABASE", None) diff --git a/ietf/blobdb/factories.py b/ietf/blobdb/factories.py new file mode 100644 index 0000000000..fdcebb2b4e --- /dev/null +++ b/ietf/blobdb/factories.py @@ -0,0 +1,13 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +import factory + +from .models import Blob + + +class BlobFactory(factory.django.DjangoModelFactory): + class Meta: + model = Blob + + name = factory.Faker("file_path") + bucket = factory.Faker("word") + content = factory.Faker("binary", length=32) # careful, default length is 1e6 diff --git a/ietf/blobdb/migrations/0001_initial.py b/ietf/blobdb/migrations/0001_initial.py new file mode 100644 index 0000000000..2fcb08e5ea --- /dev/null +++ b/ietf/blobdb/migrations/0001_initial.py @@ -0,0 +1,78 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] + + operations = [ + migrations.CreateModel( + name="Blob", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "name", + models.CharField(help_text="Name of the blob", max_length=1024), + ), + ( + "bucket", + models.CharField( + help_text="Name of the bucket containing this blob", + max_length=1024, + ), + ), + ( + "modified", + models.DateTimeField( + default=django.utils.timezone.now, + help_text="Last modification time of the blob", + ), + ), + ("content", models.BinaryField(help_text="Content of the blob")), + ( + "checksum", + models.CharField( + editable=False, + help_text="SHA-384 digest of the content", + max_length=96, + ), + ), + ( + "mtime", + models.DateTimeField( + blank=True, + default=None, + help_text="mtime associated with the blob as a filesystem object", + null=True, + ), + ), + ( + "content_type", + models.CharField( + blank=True, + help_text="content-type header value for the blob contents", + max_length=1024, + ), + ), + ], + ), + migrations.AddConstraint( + model_name="blob", + constraint=models.UniqueConstraint( + fields=("bucket", "name"), name="unique_name_per_bucket" + ), + ), + ] diff --git a/ietf/blobdb/migrations/0002_resolvedmaterial.py b/ietf/blobdb/migrations/0002_resolvedmaterial.py new file mode 100644 index 0000000000..e0ab405b11 --- /dev/null +++ b/ietf/blobdb/migrations/0002_resolvedmaterial.py @@ -0,0 +1,48 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("blobdb", "0001_initial"), + ] + + operations = [ + migrations.CreateModel( + name="ResolvedMaterial", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(help_text="Name to resolve", max_length=300)), + ( + "meeting_number", + models.CharField( + help_text="Meeting material is related to", max_length=64 + ), + ), + ( + "bucket", + models.CharField(help_text="Resolved bucket name", max_length=255), + ), + ( + "blob", + models.CharField(help_text="Resolved blob name", max_length=300), + ), + ], + ), + migrations.AddConstraint( + model_name="resolvedmaterial", + constraint=models.UniqueConstraint( + fields=("name", "meeting_number"), name="unique_name_per_meeting" + ), + ), + ] diff --git a/ietf/secr/sreq/templatetags/__init__.py b/ietf/blobdb/migrations/__init__.py similarity index 100% rename from ietf/secr/sreq/templatetags/__init__.py rename to ietf/blobdb/migrations/__init__.py diff --git a/ietf/blobdb/models.py b/ietf/blobdb/models.py new file mode 100644 index 0000000000..6dbb615fa0 --- /dev/null +++ b/ietf/blobdb/models.py @@ -0,0 +1,102 @@ +# Copyright The IETF Trust 2025-2026, All Rights Reserved +from hashlib import sha384 + +from django.db import models, transaction +from django.utils import timezone + +from .apps import get_blobdb +from .utils import queue_for_replication + + +class BlobQuerySet(models.QuerySet): + """QuerySet customized for Blob management + + Operations that bypass save() / delete() won't correctly notify watchers of changes + to the blob contents. Disallow them. + """ + + def delete(self): + raise NotImplementedError("Only deleting individual Blobs is supported") + + def bulk_create(self, *args, **kwargs): + raise NotImplementedError("Only creating individual Blobs is supported") + + def update(self, *args, **kwargs): + # n.b., update_or_create() _does_ call save() + raise NotImplementedError("Updating BlobQuerySets is not supported") + + def bulk_update(self, *args, **kwargs): + raise NotImplementedError("Updating Blobs in bulk is not supported") + + +class Blob(models.Model): + objects = BlobQuerySet.as_manager() + name = models.CharField(max_length=1024, help_text="Name of the blob") + bucket = models.CharField( + max_length=1024, help_text="Name of the bucket containing this blob" + ) + modified = models.DateTimeField( + default=timezone.now, help_text="Last modification time of the blob" + ) + content = models.BinaryField(help_text="Content of the blob") + checksum = models.CharField( + max_length=96, help_text="SHA-384 digest of the content", editable=False + ) + mtime = models.DateTimeField( + default=None, + blank=True, + null=True, + help_text="mtime associated with the blob as a filesystem object", + ) + content_type = models.CharField( + max_length=1024, + blank=True, + help_text="content-type header value for the blob contents", + ) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["bucket", "name"], name="unique_name_per_bucket" + ), + ] + + def __str__(self): + return f"{self.bucket}:{self.name}" + + def save(self, **kwargs): + db = get_blobdb() + with transaction.atomic(using=db): + self.checksum = sha384(self.content, usedforsecurity=False).hexdigest() + super().save(**kwargs) + self._emit_blob_change_event(using=db) + + def delete(self, **kwargs): + db = get_blobdb() + with transaction.atomic(using=db): + retval = super().delete(**kwargs) + self._emit_blob_change_event(using=db) + return retval + + def _emit_blob_change_event(self, using: str | None=None): + queue_for_replication(self.bucket, self.name, using=using) + + +class ResolvedMaterial(models.Model): + # A Document name can be 255 characters; allow this name to be a bit longer + name = models.CharField(max_length=300, help_text="Name to resolve") + meeting_number = models.CharField( + max_length=64, help_text="Meeting material is related to" + ) + bucket = models.CharField(max_length=255, help_text="Resolved bucket name") + blob = models.CharField(max_length=300, help_text="Resolved blob name") + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["name", "meeting_number"], name="unique_name_per_meeting" + ) + ] + + def __str__(self): + return f"{self.name}@{self.meeting_number} -> {self.bucket}:{self.blob}" diff --git a/ietf/blobdb/replication.py b/ietf/blobdb/replication.py new file mode 100644 index 0000000000..d251d3b95c --- /dev/null +++ b/ietf/blobdb/replication.py @@ -0,0 +1,178 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +import datetime +from dataclasses import dataclass +from io import BytesIO +from typing import Optional + +from django.conf import settings +from django.core.files import File +from django.core.files.storage import storages, InvalidStorageError +from django.db import connections + +from ietf.utils import log + +DEFAULT_SETTINGS = { + "ENABLED": False, + "DEST_STORAGE_PATTERN": "r2-{bucket}", + "INCLUDE_BUCKETS": (), # empty means include all + "EXCLUDE_BUCKETS": (), # empty means exclude none + "VERBOSE_LOGGING": False, +} + + +class SimpleMetadataFile(File): + def __init__(self, file, name=None): + super().__init__(file, name) + self.custom_metadata = {} + self.content_type = "" + + +def get_replication_settings(): + return DEFAULT_SETTINGS | getattr(settings, "BLOBDB_REPLICATION", {}) + + +def validate_replication_settings(): + replicator_settings = get_replication_settings() + # No extra settings allowed + unknown_settings = set(DEFAULT_SETTINGS.keys()) - set(replicator_settings.keys()) + if len(unknown_settings) > 0: + raise RuntimeError( + f"Unrecognized BLOBDB_REPLICATOR settings: {', '.join(str(unknown_settings))}" + ) + # destination storage pattern must be a string that includes {bucket} + pattern = replicator_settings["DEST_STORAGE_PATTERN"] + if not isinstance(pattern, str): + raise RuntimeError( + f"DEST_STORAGE_PATTERN must be a str, not {type(pattern).__name__}" + ) + if "{bucket}" not in pattern: + raise RuntimeError( + f"DEST_STORAGE_PATTERN must contain the substring '{{bucket}}' (found '{pattern}')" + ) + # include/exclude buckets must be list-like + include_buckets = replicator_settings["INCLUDE_BUCKETS"] + if not isinstance(include_buckets, (list, tuple, set)): + raise RuntimeError("INCLUDE_BUCKETS must be a list, tuple, or set") + exclude_buckets = replicator_settings["EXCLUDE_BUCKETS"] + if not isinstance(exclude_buckets, (list, tuple, set)): + raise RuntimeError("EXCLUDE_BUCKETS must be a list, tuple, or set") + # if we have explicit include_buckets, make sure the necessary storages exist + if len(include_buckets) > 0: + include_storages = {destination_storage_name_for(b) for b in include_buckets} + exclude_storages = {destination_storage_name_for(b) for b in exclude_buckets} + configured_storages = set(settings.STORAGES.keys()) + missing_storages = include_storages - exclude_storages - configured_storages + if len(missing_storages) > 0: + raise RuntimeError( + f"Replication requires unknown storage(s): {', '.join(missing_storages)}" + ) + + +def destination_storage_name_for(bucket: str): + pattern = get_replication_settings()["DEST_STORAGE_PATTERN"] + return pattern.format(bucket=bucket) + + +def destination_storage_for(bucket: str): + storage_name = destination_storage_name_for(bucket) + return storages[storage_name] + + +def replication_enabled(bucket: str): + replication_settings = get_replication_settings() + if not replication_settings["ENABLED"]: + return False + # Default is all buckets are included + included = ( + len(replication_settings["INCLUDE_BUCKETS"]) == 0 + or bucket in replication_settings["INCLUDE_BUCKETS"] + ) + # Default is no buckets are excluded + excluded = ( + len(replication_settings["EXCLUDE_BUCKETS"]) > 0 + and bucket in replication_settings["EXCLUDE_BUCKETS"] + ) + return included and not excluded + + +def verbose_logging_enabled(): + return bool(get_replication_settings()["VERBOSE_LOGGING"]) + + +@dataclass +class SqlBlob: + content: bytes + checksum: str + modified: datetime.datetime + mtime: Optional[datetime.datetime] + content_type: str + + +def fetch_blob_via_sql(bucket: str, name: str) -> Optional[SqlBlob]: + blobdb_connection = connections["blobdb"] + cursor = blobdb_connection.cursor() + cursor.execute( + """ + SELECT content, checksum, modified, mtime, content_type FROM blobdb_blob + WHERE bucket=%s AND name=%s LIMIT 1 + """, + [bucket, name], + ) + row = cursor.fetchone() + col_names = [col[0] for col in cursor.description] + return None if row is None else SqlBlob(**{ + col_name: row_val + for col_name, row_val in zip(col_names, row) + }) + + +def replicate_blob(bucket, name): + """Replicate a Blobdb blob to a Storage""" + if not replication_enabled(bucket): + if verbose_logging_enabled(): + log.log( + f"Not replicating {bucket}:{name} because replication is not enabled for {bucket}" + ) + return + + try: + destination_storage = destination_storage_for(bucket) + except InvalidStorageError as e: + log.log( + f"Failed to replicate {bucket}:{name} because destination storage for {bucket} is not configured" + ) + raise ReplicationError from e + + blob = fetch_blob_via_sql(bucket, name) + if blob is None: + if verbose_logging_enabled(): + log.log(f"Deleting {bucket}:{name} from replica") + try: + destination_storage.delete(name) + except Exception as e: + log.log(f"Failed to delete {bucket}:{name} from replica: {e}") + raise ReplicationError from e + else: + # Add metadata expected by the MetadataS3Storage + file_with_metadata = SimpleMetadataFile(file=BytesIO(blob.content)) + file_with_metadata.content_type = blob.content_type + file_with_metadata.custom_metadata = { + "sha384": blob.checksum, + "mtime": (blob.mtime or blob.modified).isoformat(), + } + if verbose_logging_enabled(): + log.log( + f"Saving {bucket}:{name} to replica (" + f"sha384: '{file_with_metadata.custom_metadata['sha384'][:16]}...', " + f"content_type: '{file_with_metadata.content_type}', " + f"mtime: '{file_with_metadata.custom_metadata['mtime']})" + ) + try: + destination_storage.save(name, file_with_metadata) + except Exception as e: + log.log(f"Failed to save {bucket}:{name} to replica: {e}") + raise ReplicationError from e + + +class ReplicationError(Exception): + pass diff --git a/ietf/blobdb/routers.py b/ietf/blobdb/routers.py new file mode 100644 index 0000000000..319c0fbc71 --- /dev/null +++ b/ietf/blobdb/routers.py @@ -0,0 +1,58 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +from django.apps import apps + +from .apps import BlobdbConfig, get_blobdb + + +class BlobdbStorageRouter: + """Database router for the Blobdb""" + + _app_label = None + + @property + def app_label(self): + if self._app_label is None: + for app in apps.get_app_configs(): + if isinstance(app, BlobdbConfig): + self._app_label = app.label + break + if self._app_label is None: + raise RuntimeError( + f"{BlobdbConfig.name} is not present in the Django app registry" + ) + return self._app_label + + @property + def db(self): + return get_blobdb() + + def db_for_read(self, model, **hints): + """Suggest the database that should be used for read operations for objects of type model + + Returns None if there is no suggestion. + """ + if model._meta.app_label == self.app_label: + return self.db + return None # no suggestion + + def db_for_write(self, model, **hints): + """Suggest the database that should be used for write of objects of type model + + Returns None if there is no suggestion. + """ + if model._meta.app_label == self.app_label: + return self.db + return None # no suggestion + + def allow_migrate(self, db, app_label, model_name=None, **hints): + """Determine if the migration operation is allowed to run on the database with alias db + + Return True if the operation should run, False if it shouldn’t run, or + None if the router has no opinion. + """ + if self.db is None: + return None # no opinion, use the default db + is_our_app = app_label == self.app_label + is_our_db = db == self.db + if is_our_app or is_our_db: + return is_our_app and is_our_db diff --git a/ietf/blobdb/storage.py b/ietf/blobdb/storage.py new file mode 100644 index 0000000000..e304dabc5d --- /dev/null +++ b/ietf/blobdb/storage.py @@ -0,0 +1,106 @@ +# Copyright The IETF Trust 2025-2026, All Rights Reserved +from typing import Optional + +from django.core.exceptions import SuspiciousFileOperation +from django.core.files.base import ContentFile +from django.core.files.storage import Storage +from django.db.models.functions import Length +from django.utils.deconstruct import deconstructible +from django.utils import timezone + +from ietf.utils.storage import MetadataFile +from .models import Blob +from .utils import queue_for_replication + + +class BlobFile(MetadataFile): + + def __init__(self, content, name=None, mtime=None, content_type=""): + super().__init__( + file=ContentFile(content), + name=name, + mtime=mtime, + content_type=content_type, + ) + + +@deconstructible +class BlobdbStorage(Storage): + + def __init__(self, bucket_name: Optional[str]=None): + if bucket_name is None: + raise ValueError("BlobdbStorage bucket_name must be specified") + self.bucket_name = bucket_name + + def get_queryset(self): + return Blob.objects.filter(bucket=self.bucket_name) + + def delete(self, name): + blob = self.get_queryset().filter(name=name).first() + if blob is not None: + blob.delete() + + def exists(self, name): + return self.get_queryset().filter(name=name).exists() + + def size(self, name): + sizes = ( + self.get_queryset() + .filter(name=name) + .annotate(object_size=Length("content")) + .values_list("object_size", flat=True) + ) + if len(sizes) == 0: + raise FileNotFoundError( + f"No object '{name}' exists in bucket '{self.bucket_name}'" + ) + return sizes[0] # unique constraint guarantees 0 or 1 entry + + def _open(self, name, mode="rb"): + try: + blob = self.get_queryset().get(name=name) + except Blob.DoesNotExist: + raise FileNotFoundError( + f"No object '{name}' exists in bucket '{self.bucket_name}'" + ) + return BlobFile( + content=blob.content, + name=blob.name, + mtime=blob.mtime or blob.modified, # fall back to modified time + content_type=blob.content_type, + ) + + def _save(self, name, content): + """Perform the save operation + + The storage API allows _save() to save to a different name than was requested. This method will + never do that, instead overwriting the existing blob. + """ + Blob.objects.update_or_create( + name=name, + bucket=self.bucket_name, + defaults={ + "content": content.read(), + "modified": timezone.now(), + "mtime": getattr(content, "mtime", None), + "content_type": getattr(content, "content_type", ""), + }, + ) + return name + + def get_available_name(self, name, max_length=None): + if max_length is not None and len(name) > max_length: + raise SuspiciousFileOperation( + f"BlobdbStorage only allows names up to {max_length}, but was" + f"asked to store the name '{name[:5]}...{name[-5:]} of length {len(name)}" + ) + return name # overwrite is permitted + + def force_replication(self, name: str): + """Force replication of a blob by name + + Be careful with this - replication includes replicating deletion of blobs, so + if you call it with a name that does not exist in blobdb, it will be removed + from R2 if it exists there! + """ + queue_for_replication(bucket=self.bucket_name, name=name) diff --git a/ietf/blobdb/tasks.py b/ietf/blobdb/tasks.py new file mode 100644 index 0000000000..538d415830 --- /dev/null +++ b/ietf/blobdb/tasks.py @@ -0,0 +1,17 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +import json + +from celery import shared_task + +from .replication import replicate_blob, ReplicationError + + +@shared_task( + autoretry_for=(ReplicationError,), retry_backoff=10, retry_kwargs={"max_retries": 5} +) +def pybob_the_blob_replicator_task(body: str): + request = json.loads(body) + bucket = request["bucket"] + name = request["name"] + replicate_blob(bucket, name) diff --git a/ietf/blobdb/tests.py b/ietf/blobdb/tests.py new file mode 100644 index 0000000000..0eadad0a1f --- /dev/null +++ b/ietf/blobdb/tests.py @@ -0,0 +1,80 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +import datetime + +from django.core.files.base import ContentFile + +from ietf.utils.test_utils import TestCase +from .factories import BlobFactory +from .models import Blob +from .storage import BlobFile, BlobdbStorage + + +class StorageTests(TestCase): + def test_save(self): + storage = BlobdbStorage(bucket_name="my-bucket") + timestamp = datetime.datetime( + 2025, + 3, + 17, + 1, + 2, + 3, + tzinfo=datetime.timezone.utc, + ) + # Create file to save + my_file = BlobFile( + content=b"These are my bytes.", + mtime=timestamp, + content_type="application/x-my-content-type", + ) + # save the file + saved_name = storage.save("myfile.txt", my_file) + # validate the outcome + self.assertEqual(saved_name, "myfile.txt") + blob = Blob.objects.filter(bucket="my-bucket", name="myfile.txt").first() + self.assertIsNotNone(blob) # validates bucket and name + self.assertEqual(bytes(blob.content), b"These are my bytes.") + self.assertEqual(blob.mtime, timestamp) + self.assertEqual(blob.content_type, "application/x-my-content-type") + + def test_save_naive_file(self): + storage = BlobdbStorage(bucket_name="my-bucket") + my_naive_file = ContentFile(content=b"These are my naive bytes.") + # save the file + saved_name = storage.save("myfile.txt", my_naive_file) + # validate the outcome + self.assertEqual(saved_name, "myfile.txt") + blob = Blob.objects.filter(bucket="my-bucket", name="myfile.txt").first() + self.assertIsNotNone(blob) # validates bucket and name + self.assertEqual(bytes(blob.content), b"These are my naive bytes.") + self.assertIsNone(blob.mtime) + self.assertEqual(blob.content_type, "") + + def test_open(self): + """BlobdbStorage open yields a BlobFile with specific mtime and content_type""" + mtime = datetime.datetime(2021, 1, 2, 3, 45, tzinfo=datetime.timezone.utc) + blob = BlobFactory(mtime=mtime, content_type="application/x-oh-no-you-didnt") + storage = BlobdbStorage(bucket_name=blob.bucket) + with storage.open(blob.name, "rb") as f: + self.assertTrue(isinstance(f, BlobFile)) + assert isinstance(f, BlobFile) # redundant, narrows type for linter + self.assertEqual(f.read(), bytes(blob.content)) + self.assertEqual(f.mtime, mtime) + self.assertEqual(f.content_type, "application/x-oh-no-you-didnt") + + def test_open_null_mtime(self): + """BlobdbStorage open yields a BlobFile with default mtime and content_type""" + blob = BlobFactory(content_type="application/x-oh-no-you-didnt") # does not set mtime + storage = BlobdbStorage(bucket_name=blob.bucket) + with storage.open(blob.name, "rb") as f: + self.assertTrue(isinstance(f, BlobFile)) + assert isinstance(f, BlobFile) # redundant, narrows type for linter + self.assertEqual(f.read(), bytes(blob.content)) + self.assertIsNotNone(f.mtime) + self.assertEqual(f.mtime, blob.modified) + self.assertEqual(f.content_type, "application/x-oh-no-you-didnt") + + def test_open_file_not_found(self): + storage = BlobdbStorage(bucket_name="not-a-bucket") + with self.assertRaises(FileNotFoundError): + storage.open("definitely/not-a-file.txt") diff --git a/ietf/blobdb/utils.py b/ietf/blobdb/utils.py new file mode 100644 index 0000000000..93f8f2f521 --- /dev/null +++ b/ietf/blobdb/utils.py @@ -0,0 +1,32 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +import json +from functools import partial + +from django.db import transaction + +from ietf.blobdb.replication import replication_enabled +from ietf.blobdb.tasks import pybob_the_blob_replicator_task + + +def queue_for_replication(bucket: str, name: str, using: str | None=None): + """Queue a blob for replication + + This is private to the blobdb app. Do not call it directly from other apps. + """ + if not replication_enabled(bucket): + return + + # For now, fire a celery task we've arranged to guarantee in-order processing. + # Later becomes pushing an event onto a queue to a dedicated worker. + transaction.on_commit( + partial( + pybob_the_blob_replicator_task.delay, + json.dumps( + { + "name": name, + "bucket": bucket, + } + ) + ), + using=using, + ) diff --git a/ietf/community/tests.py b/ietf/community/tests.py index 9bd7789958..04f1433d61 100644 --- a/ietf/community/tests.py +++ b/ietf/community/tests.py @@ -1,18 +1,25 @@ # Copyright The IETF Trust 2016-2023, All Rights Reserved # -*- coding: utf-8 -*- -import mock +from unittest import mock from pyquery import PyQuery from django.test.utils import override_settings from django.urls import reverse as urlreverse +from lxml import etree -import debug # pyflakes:ignore +import debug # pyflakes:ignore from ietf.community.models import CommunityList, SearchRule, EmailSubscription from ietf.community.signals import notify_of_event -from ietf.community.utils import docs_matching_community_list_rule, community_list_rules_matching_doc -from ietf.community.utils import reset_name_contains_index_for_rule, notify_event_to_subscribers +from ietf.community.utils import ( + docs_matching_community_list_rule, + community_list_rules_matching_doc, +) +from ietf.community.utils import ( + reset_name_contains_index_for_rule, + notify_event_to_subscribers, +) from ietf.community.tasks import notify_event_to_subscribers_task import ietf.community.views from ietf.group.models import Group @@ -26,35 +33,80 @@ from ietf.group.factories import GroupFactory, RoleFactory from ietf.person.factories import PersonFactory, EmailFactory, AliasFactory + class CommunityListTests(TestCase): def test_rule_matching(self): - plain = PersonFactory(user__username='plain') - ad = Person.objects.get(user__username='ad') + plain = PersonFactory(user__username="plain") + ad = Person.objects.get(user__username="ad") draft = WgDraftFactory( - group__parent=Group.objects.get(acronym='farfut' ), + group__parent=Group.objects.get(acronym="farfut"), authors=[ad], ad=ad, shepherd=plain.email(), - states=[('draft-iesg','lc'),('draft','active')], + states=[("draft-iesg", "lc"), ("draft", "active")], ) clist = CommunityList.objects.create(person=plain) - rule_group = SearchRule.objects.create(rule_type="group", group=draft.group, state=State.objects.get(type="draft", slug="active"), community_list=clist) - rule_group_rfc = SearchRule.objects.create(rule_type="group_rfc", group=draft.group, state=State.objects.get(type="rfc", slug="published"), community_list=clist) - rule_area = SearchRule.objects.create(rule_type="area", group=draft.group.parent, state=State.objects.get(type="draft", slug="active"), community_list=clist) + rule_group = SearchRule.objects.create( + rule_type="group", + group=draft.group, + state=State.objects.get(type="draft", slug="active"), + community_list=clist, + ) + rule_group_rfc = SearchRule.objects.create( + rule_type="group_rfc", + group=draft.group, + state=State.objects.get(type="rfc", slug="published"), + community_list=clist, + ) + rule_area = SearchRule.objects.create( + rule_type="area", + group=draft.group.parent, + state=State.objects.get(type="draft", slug="active"), + community_list=clist, + ) - rule_state_iesg = SearchRule.objects.create(rule_type="state_iesg", state=State.objects.get(type="draft-iesg", slug="lc"), community_list=clist) + rule_state_iesg = SearchRule.objects.create( + rule_type="state_iesg", + state=State.objects.get(type="draft-iesg", slug="lc"), + community_list=clist, + ) - rule_author = SearchRule.objects.create(rule_type="author", state=State.objects.get(type="draft", slug="active"), person=Person.objects.filter(documentauthor__document=draft).first(), community_list=clist) + rule_author = SearchRule.objects.create( + rule_type="author", + state=State.objects.get(type="draft", slug="active"), + person=Person.objects.filter(documentauthor__document=draft).first(), + community_list=clist, + ) - rule_ad = SearchRule.objects.create(rule_type="ad", state=State.objects.get(type="draft", slug="active"), person=draft.ad, community_list=clist) + rule_ad = SearchRule.objects.create( + rule_type="ad", + state=State.objects.get(type="draft", slug="active"), + person=draft.ad, + community_list=clist, + ) - rule_shepherd = SearchRule.objects.create(rule_type="shepherd", state=State.objects.get(type="draft", slug="active"), person=draft.shepherd.person, community_list=clist) + rule_shepherd = SearchRule.objects.create( + rule_type="shepherd", + state=State.objects.get(type="draft", slug="active"), + person=draft.shepherd.person, + community_list=clist, + ) - rule_group_exp = SearchRule.objects.create(rule_type="group_exp", group=draft.group, state=State.objects.get(type="draft", slug="expired"), community_list=clist) + rule_group_exp = SearchRule.objects.create( + rule_type="group_exp", + group=draft.group, + state=State.objects.get(type="draft", slug="expired"), + community_list=clist, + ) - rule_name_contains = SearchRule.objects.create(rule_type="name_contains", state=State.objects.get(type="draft", slug="active"), text="draft-.*" + "-".join(draft.name.split("-")[2:]), community_list=clist) + rule_name_contains = SearchRule.objects.create( + rule_type="name_contains", + state=State.objects.get(type="draft", slug="active"), + text="draft-.*" + "-".join(draft.name.split("-")[2:]), + community_list=clist, + ) reset_name_contains_index_for_rule(rule_name_contains) # doc -> rules @@ -71,29 +123,44 @@ def test_rule_matching(self): # rule -> docs self.assertTrue(draft in list(docs_matching_community_list_rule(rule_group))) - self.assertTrue(draft not in list(docs_matching_community_list_rule(rule_group_rfc))) + self.assertTrue( + draft not in list(docs_matching_community_list_rule(rule_group_rfc)) + ) self.assertTrue(draft in list(docs_matching_community_list_rule(rule_area))) - self.assertTrue(draft in list(docs_matching_community_list_rule(rule_state_iesg))) + self.assertTrue( + draft in list(docs_matching_community_list_rule(rule_state_iesg)) + ) self.assertTrue(draft in list(docs_matching_community_list_rule(rule_author))) self.assertTrue(draft in list(docs_matching_community_list_rule(rule_ad))) self.assertTrue(draft in list(docs_matching_community_list_rule(rule_shepherd))) - self.assertTrue(draft in list(docs_matching_community_list_rule(rule_name_contains))) - self.assertTrue(draft not in list(docs_matching_community_list_rule(rule_group_exp))) + self.assertTrue( + draft in list(docs_matching_community_list_rule(rule_name_contains)) + ) + self.assertTrue( + draft not in list(docs_matching_community_list_rule(rule_group_exp)) + ) - draft.set_state(State.objects.get(type='draft', slug='expired')) + draft.set_state(State.objects.get(type="draft", slug="expired")) # doc -> rules matching_rules = list(community_list_rules_matching_doc(draft)) self.assertTrue(rule_group_exp in matching_rules) # rule -> docs - self.assertTrue(draft in list(docs_matching_community_list_rule(rule_group_exp))) + self.assertTrue( + draft in list(docs_matching_community_list_rule(rule_group_exp)) + ) def test_view_list_duplicates(self): - person = PersonFactory(name="John Q. Public", user__username="bazquux@example.com") + person = PersonFactory( + name="John Q. Public", user__username="bazquux@example.com" + ) PersonFactory(name="John Q. Public", user__username="foobar@example.com") - url = urlreverse(ietf.community.views.view_list, kwargs={ "email_or_name": person.plain_name()}) + url = urlreverse( + ietf.community.views.view_list, + kwargs={"email_or_name": person.plain_name()}, + ) r = self.client.get(url) self.assertEqual(r.status_code, 404) @@ -104,20 +171,23 @@ def complex_person(self, *args, **kwargs): return person def email_or_name_set(self, person): - return [e for e in Email.objects.filter(person=person)] + \ - [a for a in Alias.objects.filter(person=person)] + return [e for e in Email.objects.filter(person=person)] + [ + a for a in Alias.objects.filter(person=person) + ] def do_view_list_test(self, person): draft = WgDraftFactory() # without list for id in self.email_or_name_set(person): - url = urlreverse(ietf.community.views.view_list, kwargs={ "email_or_name": id }) + url = urlreverse( + ietf.community.views.view_list, kwargs={"email_or_name": id} + ) r = self.client.get(url) self.assertEqual(r.status_code, 200, msg=f"id='{id}', url='{url}'") # with list clist = CommunityList.objects.create(person=person) - if not draft in clist.added_docs.all(): + if draft not in clist.added_docs.all(): clist.added_docs.add(draft) SearchRule.objects.create( community_list=clist, @@ -126,31 +196,37 @@ def do_view_list_test(self, person): text="test", ) for id in self.email_or_name_set(person): - url = urlreverse(ietf.community.views.view_list, kwargs={ "email_or_name": id }) + url = urlreverse( + ietf.community.views.view_list, kwargs={"email_or_name": id} + ) r = self.client.get(url) self.assertEqual(r.status_code, 200, msg=f"id='{id}', url='{url}'") self.assertContains(r, draft.name) def test_view_list(self): - person = self.complex_person(user__username='plain') + person = self.complex_person(user__username="plain") self.do_view_list_test(person) - + def test_view_list_without_active_email(self): - person = self.complex_person(user__username='plain') + person = self.complex_person(user__username="plain") person.email_set.update(active=False) self.do_view_list_test(person) def test_manage_personal_list(self): - person = self.complex_person(user__username='plain') - ad = Person.objects.get(user__username='ad') + person = self.complex_person(user__username="plain") + ad = Person.objects.get(user__username="ad") draft = WgDraftFactory(authors=[ad]) - url = urlreverse(ietf.community.views.manage_list, kwargs={ "email_or_name": person.email() }) + url = urlreverse( + ietf.community.views.manage_list, kwargs={"email_or_name": person.email()} + ) login_testing_unauthorized(self, "plain", url) for id in self.email_or_name_set(person): - url = urlreverse(ietf.community.views.manage_list, kwargs={ "email_or_name": id }) - r = self.client.get(url, user='plain') + url = urlreverse( + ietf.community.views.manage_list, kwargs={"email_or_name": id} + ) + r = self.client.get(url, user="plain") self.assertEqual(r.status_code, 200, msg=f"id='{id}', url='{url}'") # We can't call post() with follow=True because that 404's if @@ -158,11 +234,13 @@ def test_manage_personal_list(self): # apparently re-encodes the already-encoded url. def follow(r): redirect_url = r.url or url - return self.client.get(redirect_url, user='plain') + return self.client.get(redirect_url, user="plain") # add document - self.assertContains(r, 'add_document') - r = self.client.post(url, {'action': 'add_documents', 'documents': draft.pk}) + self.assertContains(r, "add_document") + r = self.client.post( + url, {"action": "add_documents", "documents": draft.pk} + ) self.assertEqual(r.status_code, 302, msg=f"id='{id}', url='{url}'") clist = CommunityList.objects.get(person__user__username="plain") self.assertTrue(clist.added_docs.filter(pk=draft.pk)) @@ -170,8 +248,10 @@ def follow(r): self.assertContains(r, draft.name, status_code=200) # remove document - self.assertContains(r, 'remove_document_%s' % draft.pk) - r = self.client.post(url, {'action': 'remove_document', 'document': draft.pk}) + self.assertContains(r, "remove_document_%s" % draft.pk) + r = self.client.post( + url, {"action": "remove_document", "document": draft.pk} + ) self.assertEqual(r.status_code, 302, msg=f"id='{id}', url='{url}'") clist = CommunityList.objects.get(person__user__username="plain") self.assertTrue(not clist.added_docs.filter(pk=draft.pk)) @@ -179,23 +259,37 @@ def follow(r): self.assertNotContains(r, draft.name, status_code=200) # add rule - r = self.client.post(url, { - "action": "add_rule", - "rule_type": "author_rfc", - "author_rfc-person": Person.objects.filter(documentauthor__document=draft).first().pk, - "author_rfc-state": State.objects.get(type="rfc", slug="published").pk, - }) + r = self.client.post( + url, + { + "action": "add_rule", + "rule_type": "author_rfc", + "author_rfc-person": Person.objects.filter( + documentauthor__document=draft + ) + .first() + .pk, + "author_rfc-state": State.objects.get( + type="rfc", slug="published" + ).pk, + }, + ) self.assertEqual(r.status_code, 302, msg=f"id='{id}', url='{url}'") clist = CommunityList.objects.get(person__user__username="plain") self.assertTrue(clist.searchrule_set.filter(rule_type="author_rfc")) # add name_contains rule - r = self.client.post(url, { - "action": "add_rule", - "rule_type": "name_contains", - "name_contains-text": "draft.*mars", - "name_contains-state": State.objects.get(type="draft", slug="active").pk, - }) + r = self.client.post( + url, + { + "action": "add_rule", + "rule_type": "name_contains", + "name_contains-text": "draft.*mars", + "name_contains-state": State.objects.get( + type="draft", slug="active" + ).pk, + }, + ) self.assertEqual(r.status_code, 302, msg=f"id='{id}', url='{url}'") clist = CommunityList.objects.get(person__user__username="plain") self.assertTrue(clist.searchrule_set.filter(rule_type="name_contains")) @@ -205,22 +299,31 @@ def follow(r): self.assertEqual(r.status_code, 200, msg=f"id='{id}', url='{url}'") rule = clist.searchrule_set.filter(rule_type="author_rfc").first() q = PyQuery(r.content) - self.assertEqual(len(q('#r%s' % rule.pk)), 1) + self.assertEqual(len(q("#r%s" % rule.pk)), 1) # remove rule - r = self.client.post(url, { - "action": "remove_rule", - "rule": rule.pk, - }) + r = self.client.post( + url, + { + "action": "remove_rule", + "rule": rule.pk, + }, + ) clist = CommunityList.objects.get(person__user__username="plain") self.assertTrue(not clist.searchrule_set.filter(rule_type="author_rfc")) def test_manage_group_list(self): - draft = WgDraftFactory(group__acronym='mars') - RoleFactory(group__acronym='mars',name_id='chair',person=PersonFactory(user__username='marschairman')) + draft = WgDraftFactory(group__acronym="mars") + RoleFactory( + group__acronym="mars", + name_id="chair", + person=PersonFactory(user__username="marschairman"), + ) - url = urlreverse(ietf.community.views.manage_list, kwargs={ "acronym": draft.group.acronym }) + url = urlreverse( + ietf.community.views.manage_list, kwargs={"acronym": draft.group.acronym} + ) setup_default_community_list_for_group(draft.group) login_testing_unauthorized(self, "marschairman", url) @@ -229,27 +332,41 @@ def test_manage_group_list(self): self.assertEqual(r.status_code, 200) # Verify GET also works with non-WG and RG groups - for gtype in ['area','program']: + for gtype in ["area", "program"]: g = GroupFactory.create(type_id=gtype) # make sure the group's features have been initialized to improve coverage - _ = g.features # pyflakes:ignore + _ = g.features # pyflakes:ignore p = PersonFactory() - g.role_set.create(name_id={'area':'ad','program':'lead'}[gtype],person=p, email=p.email()) - url = urlreverse(ietf.community.views.manage_list, kwargs={ "acronym": g.acronym }) + g.role_set.create( + name_id={"area": "ad", "program": "lead"}[gtype], + person=p, + email=p.email(), + ) + url = urlreverse( + ietf.community.views.manage_list, kwargs={"acronym": g.acronym} + ) setup_default_community_list_for_group(g) - self.client.login(username=p.user.username,password=p.user.username+"+password") + self.client.login( + username=p.user.username, password=p.user.username + "+password" + ) r = self.client.get(url) self.assertEqual(r.status_code, 200) def test_track_untrack_document(self): - person = self.complex_person(user__username='plain') + person = self.complex_person(user__username="plain") draft = WgDraftFactory() - url = urlreverse(ietf.community.views.track_document, kwargs={ "email_or_name": person.email(), "name": draft.name }) + url = urlreverse( + ietf.community.views.track_document, + kwargs={"email_or_name": person.email(), "name": draft.name}, + ) login_testing_unauthorized(self, "plain", url) for id in self.email_or_name_set(person): - url = urlreverse(ietf.community.views.track_document, kwargs={ "email_or_name": id, "name": draft.name }) + url = urlreverse( + ietf.community.views.track_document, + kwargs={"email_or_name": id, "name": draft.name}, + ) # track r = self.client.get(url) @@ -261,7 +378,10 @@ def test_track_untrack_document(self): self.assertEqual(list(clist.added_docs.all()), [draft]) # untrack - url = urlreverse(ietf.community.views.untrack_document, kwargs={ "email_or_name": id, "name": draft.name }) + url = urlreverse( + ietf.community.views.untrack_document, + kwargs={"email_or_name": id, "name": draft.name}, + ) r = self.client.get(url) self.assertEqual(r.status_code, 200, msg=f"id='{id}', url='{url}'") @@ -271,36 +391,47 @@ def test_track_untrack_document(self): self.assertEqual(list(clist.added_docs.all()), []) def test_track_untrack_document_through_ajax(self): - person = self.complex_person(user__username='plain') + person = self.complex_person(user__username="plain") draft = WgDraftFactory() - url = urlreverse(ietf.community.views.track_document, kwargs={ "email_or_name": person.email(), "name": draft.name }) + url = urlreverse( + ietf.community.views.track_document, + kwargs={"email_or_name": person.email(), "name": draft.name}, + ) login_testing_unauthorized(self, "plain", url) for id in self.email_or_name_set(person): - url = urlreverse(ietf.community.views.track_document, kwargs={ "email_or_name": id, "name": draft.name }) + url = urlreverse( + ietf.community.views.track_document, + kwargs={"email_or_name": id, "name": draft.name}, + ) # track - r = self.client.post(url, HTTP_X_REQUESTED_WITH='XMLHttpRequest') + r = self.client.post(url, HTTP_X_REQUESTED_WITH="XMLHttpRequest") self.assertEqual(r.status_code, 200, msg=f"id='{id}', url='{url}'") self.assertEqual(r.json()["success"], True) clist = CommunityList.objects.get(person__user__username="plain") self.assertEqual(list(clist.added_docs.all()), [draft]) # untrack - url = urlreverse(ietf.community.views.untrack_document, kwargs={ "email_or_name": id, "name": draft.name }) - r = self.client.post(url, HTTP_X_REQUESTED_WITH='XMLHttpRequest') + url = urlreverse( + ietf.community.views.untrack_document, + kwargs={"email_or_name": id, "name": draft.name}, + ) + r = self.client.post(url, HTTP_X_REQUESTED_WITH="XMLHttpRequest") self.assertEqual(r.status_code, 200, msg=f"id='{id}', url='{url}'") self.assertEqual(r.json()["success"], True) clist = CommunityList.objects.get(person__user__username="plain") self.assertEqual(list(clist.added_docs.all()), []) def test_csv(self): - person = self.complex_person(user__username='plain') + person = self.complex_person(user__username="plain") draft = WgDraftFactory() for id in self.email_or_name_set(person): - url = urlreverse(ietf.community.views.export_to_csv, kwargs={ "email_or_name": id }) + url = urlreverse( + ietf.community.views.export_to_csv, kwargs={"email_or_name": id} + ) # without list r = self.client.get(url) @@ -308,7 +439,7 @@ def test_csv(self): # with list clist = CommunityList.objects.create(person=person) - if not draft in clist.added_docs.all(): + if draft not in clist.added_docs.all(): clist.added_docs.add(draft) SearchRule.objects.create( community_list=clist, @@ -324,7 +455,9 @@ def test_csv(self): def test_csv_for_group(self): draft = WgDraftFactory() - url = urlreverse(ietf.community.views.export_to_csv, kwargs={ "acronym": draft.group.acronym }) + url = urlreverse( + ietf.community.views.export_to_csv, kwargs={"acronym": draft.group.acronym} + ) setup_default_community_list_for_group(draft.group) @@ -333,11 +466,11 @@ def test_csv_for_group(self): self.assertEqual(r.status_code, 200) def test_feed(self): - person = self.complex_person(user__username='plain') + person = self.complex_person(user__username="plain") draft = WgDraftFactory() for id in self.email_or_name_set(person): - url = urlreverse(ietf.community.views.feed, kwargs={ "email_or_name": id }) + url = urlreverse(ietf.community.views.feed, kwargs={"email_or_name": id}) # without list r = self.client.get(url) @@ -345,7 +478,7 @@ def test_feed(self): # with list clist = CommunityList.objects.create(person=person) - if not draft in clist.added_docs.all(): + if draft not in clist.added_docs.all(): clist.added_docs.add(draft) SearchRule.objects.create( community_list=clist, @@ -357,31 +490,53 @@ def test_feed(self): self.assertEqual(r.status_code, 200, msg=f"id='{id}', url='{url}'") self.assertContains(r, draft.name) + # test atom xml + xml = etree.fromstring(r.content) + ns = {"atom": "http://www.w3.org/2005/Atom"} + updated = xml.xpath("/atom:feed/atom:updated", namespaces=ns)[0].text + entries = xml.xpath("/atom:feed/atom:entry", namespaces=ns) + self.assertIn("+00:00", updated) # RFC 3339 compatible UTC TZ + for entry in entries: + updated = entry.xpath("atom:updated", namespaces=ns)[0].text + published = entry.xpath("atom:published", namespaces=ns)[0].text + entry_id = entry.xpath("atom:id", namespaces=ns)[0].text + self.assertIn("+00:00", updated) + self.assertIn("+00:00", published) + self.assertIn( + "urn:datatracker-ietf-org:event:", entry_id + ) # atom:entry:id must be a valid URN + # only significant r = self.client.get(url + "?significant=1") self.assertEqual(r.status_code, 200, msg=f"id='{id}', url='{url}'") - self.assertNotContains(r, '') + self.assertNotContains(r, "") def test_feed_for_group(self): draft = WgDraftFactory() - url = urlreverse(ietf.community.views.feed, kwargs={ "acronym": draft.group.acronym }) + url = urlreverse( + ietf.community.views.feed, kwargs={"acronym": draft.group.acronym} + ) setup_default_community_list_for_group(draft.group) # test GET, rest is tested with personal list r = self.client.get(url) self.assertEqual(r.status_code, 200) - + def test_subscription(self): - person = self.complex_person(user__username='plain') + person = self.complex_person(user__username="plain") draft = WgDraftFactory() - url = urlreverse(ietf.community.views.subscription, kwargs={ "email_or_name": person.email() }) + url = urlreverse( + ietf.community.views.subscription, kwargs={"email_or_name": person.email()} + ) login_testing_unauthorized(self, "plain", url) for id in self.email_or_name_set(person): - url = urlreverse(ietf.community.views.subscription, kwargs={ "email_or_name": id }) + url = urlreverse( + ietf.community.views.subscription, kwargs={"email_or_name": id} + ) # subscription without list r = self.client.get(url) @@ -389,7 +544,7 @@ def test_subscription(self): # subscription with list clist = CommunityList.objects.create(person=person) - if not draft in clist.added_docs.all(): + if draft not in clist.added_docs.all(): clist.added_docs.add(draft) SearchRule.objects.create( community_list=clist, @@ -399,29 +554,49 @@ def test_subscription(self): ) for email in Email.objects.filter(person=person): - url = urlreverse(ietf.community.views.subscription, kwargs={ "email_or_name": email }) + url = urlreverse( + ietf.community.views.subscription, kwargs={"email_or_name": email} + ) r = self.client.get(url) self.assertEqual(r.status_code, 200) # subscribe - r = self.client.post(url, { "email": email.pk, "notify_on": "significant", "action": "subscribe" }) + r = self.client.post( + url, + {"email": email.pk, "notify_on": "significant", "action": "subscribe"}, + ) self.assertEqual(r.status_code, 302) - subscription = EmailSubscription.objects.filter(community_list=clist, email=email, notify_on="significant").first() + subscription = EmailSubscription.objects.filter( + community_list=clist, email=email, notify_on="significant" + ).first() self.assertTrue(subscription) # delete subscription - r = self.client.post(url, { "subscription_id": subscription.pk, "action": "unsubscribe" }) + r = self.client.post( + url, {"subscription_id": subscription.pk, "action": "unsubscribe"} + ) self.assertEqual(r.status_code, 302) - self.assertEqual(EmailSubscription.objects.filter(community_list=clist, email=email, notify_on="significant").count(), 0) + self.assertEqual( + EmailSubscription.objects.filter( + community_list=clist, email=email, notify_on="significant" + ).count(), + 0, + ) def test_subscription_for_group(self): - draft = WgDraftFactory(group__acronym='mars') - RoleFactory(group__acronym='mars',name_id='chair',person=PersonFactory(user__username='marschairman')) + draft = WgDraftFactory(group__acronym="mars") + RoleFactory( + group__acronym="mars", + name_id="chair", + person=PersonFactory(user__username="marschairman"), + ) - url = urlreverse(ietf.community.views.subscription, kwargs={ "acronym": draft.group.acronym }) + url = urlreverse( + ietf.community.views.subscription, kwargs={"acronym": draft.group.acronym} + ) setup_default_community_list_for_group(draft.group) @@ -434,7 +609,7 @@ def test_subscription_for_group(self): @mock.patch("ietf.community.signals.notify_of_event") def test_notification_signal_receiver(self, mock_notify_of_event): """Saving a newly created DocEvent should notify subscribers - + This implicitly tests that notify_of_event_receiver is hooked up to the post_save signal. """ # Arbitrary model that's not a DocEvent @@ -442,19 +617,26 @@ def test_notification_signal_receiver(self, mock_notify_of_event): mock_notify_of_event.reset_mock() # clear any calls that resulted from the factories person.save() self.assertFalse(mock_notify_of_event.called) - + # build a DocEvent that is not yet persisted doc = DocumentFactory() event = DocEventFactory.build(by=person, doc=doc) # builds but does not save... mock_notify_of_event.reset_mock() # clear any calls that resulted from the factories event.save() - self.assertEqual(mock_notify_of_event.call_count, 1, "notify_task should be run on creation of DocEvent") + self.assertEqual( + mock_notify_of_event.call_count, + 1, + "notify_task should be run on creation of DocEvent", + ) self.assertEqual(mock_notify_of_event.call_args, mock.call(event)) - # save the existing DocEvent and see that no notification is sent + # save the existing DocEvent and see that no notification is sent mock_notify_of_event.reset_mock() event.save() - self.assertFalse(mock_notify_of_event.called, "notify_task should not be run save of on existing DocEvent") + self.assertFalse( + mock_notify_of_event.called, + "notify_task should not be run save of on existing DocEvent", + ) # Mock out the on_commit call so we can tell whether the task was actually queued @mock.patch("ietf.submit.views.transaction.on_commit", side_effect=lambda x: x()) @@ -468,7 +650,10 @@ def test_notify_of_event(self, mock_notify_task, mock_on_commit): # under test does not make this call when in "test" mode with override_settings(SERVER_MODE="not-test"): notify_of_event(event) - self.assertTrue(mock_notify_task.delay.called, "notify_task should run for a DocEvent on a draft") + self.assertTrue( + mock_notify_task.delay.called, + "notify_task should run for a DocEvent on a draft", + ) mock_notify_task.reset_mock() event.skip_community_list_notification = True @@ -476,22 +661,28 @@ def test_notify_of_event(self, mock_notify_task, mock_on_commit): # under test does not make this call when in "test" mode with override_settings(SERVER_MODE="not-test"): notify_of_event(event) - self.assertFalse(mock_notify_task.delay.called, "notify_task should not run when skip_community_list_notification is set") + self.assertFalse( + mock_notify_task.delay.called, + "notify_task should not run when skip_community_list_notification is set", + ) event = DocEventFactory.build(by=person, doc=DocumentFactory(type_id="rfc")) # be careful overriding SERVER_MODE - we do it here because the method # under test does not make this call when in "test" mode with override_settings(SERVER_MODE="not-test"): notify_of_event(event) - self.assertFalse(mock_notify_task.delay.called, "notify_task should not run on a document with type 'rfc'") + self.assertFalse( + mock_notify_task.delay.called, + "notify_task should not run on a document with type 'rfc'", + ) @mock.patch("ietf.utils.mail.send_mail_text") def test_notify_event_to_subscribers(self, mock_send_mail_text): - person = PersonFactory(user__username='plain') + person = PersonFactory(user__username="plain") draft = WgDraftFactory() clist = CommunityList.objects.create(person=person) - if not draft in clist.added_docs.all(): + if draft not in clist.added_docs.all(): clist.added_docs.add(draft) sub_to_significant = EmailSubscription.objects.create( @@ -522,11 +713,13 @@ def test_notify_event_to_subscribers(self, mock_send_mail_text): mock_send_mail_text.reset_mock() notify_event_to_subscribers(event) self.assertEqual(mock_send_mail_text.call_count, 2) - addresses = [call_args[0][1] for call_args in mock_send_mail_text.call_args_list] + addresses = [ + call_args[0][1] for call_args in mock_send_mail_text.call_args_list + ] subjects = {call_args[0][3] for call_args in mock_send_mail_text.call_args_list} contents = {call_args[0][4] for call_args in mock_send_mail_text.call_args_list} self.assertCountEqual( - addresses, + addresses, [sub_to_significant.email.address, sub_to_all.email.address], ) self.assertEqual(len(subjects), 1) @@ -545,4 +738,3 @@ def test_notify_event_to_subscribers_task(self, mock_notify): d.delete() notify_event_to_subscribers_task(event_id=d.pk) self.assertFalse(mock_notify.called) - diff --git a/ietf/community/utils.py b/ietf/community/utils.py index f23e8d26ab..b6137095ef 100644 --- a/ietf/community/utils.py +++ b/ietf/community/utils.py @@ -72,8 +72,10 @@ def docs_matching_community_list_rule(rule): return docs.filter(group=rule.group_id) elif rule.rule_type.startswith("state_"): return docs - elif rule.rule_type in ["author", "author_rfc"]: + elif rule.rule_type == "author": return docs.filter(documentauthor__person=rule.person) + elif rule.rule_type == "author_rfc": + return docs.filter(Q(rfcauthor__person=rule.person)|Q(rfcauthor__isnull=True,documentauthor__person=rule.person)) elif rule.rule_type == "ad": return docs.filter(ad=rule.person) elif rule.rule_type == "shepherd": @@ -122,9 +124,16 @@ def community_list_rules_matching_doc(doc): # author rules if doc.type_id == "rfc": + has_rfcauthors = doc.rfcauthor_set.exists() rules |= SearchRule.objects.filter( rule_type="author_rfc", - person__in=list(Person.objects.filter(documentauthor__document=doc)), + person__in=list( + Person.objects.filter( + Q(rfcauthor__document=doc) + if has_rfcauthors + else Q(documentauthor__document=doc) + ) + ), ) else: rules |= SearchRule.objects.filter( diff --git a/ietf/community/views.py b/ietf/community/views.py index 923ec556f3..08b1c24fe5 100644 --- a/ietf/community/views.py +++ b/ietf/community/views.py @@ -13,13 +13,24 @@ from django.utils import timezone from django.utils.html import strip_tags -import debug # pyflakes:ignore +import debug # pyflakes:ignore from ietf.community.models import CommunityList, EmailSubscription, SearchRule -from ietf.community.forms import SearchRuleTypeForm, SearchRuleForm, AddDocumentsForm, SubscriptionForm +from ietf.community.forms import ( + SearchRuleTypeForm, + SearchRuleForm, + AddDocumentsForm, + SubscriptionForm, +) from ietf.community.utils import can_manage_community_list -from ietf.community.utils import docs_tracked_by_community_list, docs_matching_community_list_rule -from ietf.community.utils import states_of_significant_change, reset_name_contains_index_for_rule +from ietf.community.utils import ( + docs_tracked_by_community_list, + docs_matching_community_list_rule, +) +from ietf.community.utils import ( + states_of_significant_change, + reset_name_contains_index_for_rule, +) from ietf.group.models import Group from ietf.doc.models import DocEvent, Document from ietf.doc.utils_search import prepare_document_table @@ -31,43 +42,61 @@ def lookup_community_list(request, email_or_name=None, acronym=None): """Finds a CommunityList for a person or group - + Instantiates an unsaved CommunityList if one is not found. - + If the person or group cannot be found and uniquely identified, raises an Http404 exception """ assert email_or_name or acronym if acronym: group = get_object_or_404(Group, acronym=acronym) - clist = CommunityList.objects.filter(group=group).first() or CommunityList(group=group) + clist = CommunityList.objects.filter(group=group).first() or CommunityList( + group=group + ) else: persons = lookup_persons(email_or_name) if len(persons) > 1: - if hasattr(request.user, 'person') and request.user.person in persons: + if hasattr(request.user, "person") and request.user.person in persons: person = request.user.person else: - raise Http404(f"Unable to identify the CommunityList for {email_or_name}") + raise Http404( + f"Unable to identify the CommunityList for {email_or_name}" + ) else: person = persons[0] - clist = CommunityList.objects.filter(person=person).first() or CommunityList(person=person) + clist = CommunityList.objects.filter(person=person).first() or CommunityList( + person=person + ) return clist + def view_list(request, email_or_name=None): clist = lookup_community_list(request, email_or_name) # may raise Http404 docs = docs_tracked_by_community_list(clist) docs, meta = prepare_document_table(request, docs, request.GET) - subscribed = request.user.is_authenticated and (EmailSubscription.objects.none() if clist.pk is None else EmailSubscription.objects.filter(community_list=clist, email__person__user=request.user)) + subscribed = request.user.is_authenticated and ( + EmailSubscription.objects.none() + if clist.pk is None + else EmailSubscription.objects.filter( + community_list=clist, email__person__user=request.user + ) + ) + + return render( + request, + "community/view_list.html", + { + "clist": clist, + "docs": docs, + "meta": meta, + "can_manage_list": can_manage_community_list(request.user, clist), + "subscribed": subscribed, + "email_or_name": email_or_name, + }, + ) - return render(request, 'community/view_list.html', { - 'clist': clist, - 'docs': docs, - 'meta': meta, - 'can_manage_list': can_manage_community_list(request.user, clist), - 'subscribed': subscribed, - "email_or_name": email_or_name, - }) @login_required @ignore_view_kwargs("group_type") @@ -79,24 +108,24 @@ def manage_list(request, email_or_name=None, acronym=None): if not can_manage_community_list(request.user, clist): permission_denied(request, "You do not have permission to access this view") - action = request.POST.get('action') + action = request.POST.get("action") - if request.method == 'POST' and action == 'add_documents': + if request.method == "POST" and action == "add_documents": add_doc_form = AddDocumentsForm(request.POST) if add_doc_form.is_valid(): if clist.pk is None: clist.save() - for d in add_doc_form.cleaned_data['documents']: - if not d in clist.added_docs.all(): + for d in add_doc_form.cleaned_data["documents"]: + if d not in clist.added_docs.all(): clist.added_docs.add(d) return HttpResponseRedirect("") else: add_doc_form = AddDocumentsForm() - if request.method == 'POST' and action == 'remove_document': - document_id = request.POST.get('document') + if request.method == "POST" and action == "remove_document": + document_id = request.POST.get("document") if clist.pk is not None and document_id: document = get_object_or_404(clist.added_docs, id=document_id) clist.added_docs.remove(document) @@ -104,16 +133,16 @@ def manage_list(request, email_or_name=None, acronym=None): return HttpResponseRedirect("") rule_form = None - if request.method == 'POST' and action == 'add_rule': + if request.method == "POST" and action == "add_rule": rule_type_form = SearchRuleTypeForm(request.POST) if rule_type_form.is_valid(): - rule_type = rule_type_form.cleaned_data['rule_type'] + rule_type = rule_type_form.cleaned_data["rule_type"] if rule_type: rule_form = SearchRuleForm(clist, rule_type, request.POST) if rule_form.is_valid(): if clist.pk is None: clist.save() - + rule = rule_form.save(commit=False) rule.community_list = clist rule.rule_type = rule_type @@ -125,8 +154,8 @@ def manage_list(request, email_or_name=None, acronym=None): else: rule_type_form = SearchRuleTypeForm() - if request.method == 'POST' and action == 'remove_rule': - rule_pk = request.POST.get('rule') + if request.method == "POST" and action == "remove_rule": + rule_pk = request.POST.get("rule") if clist.pk is not None and rule_pk: rule = get_object_or_404(SearchRule, pk=rule_pk, community_list=clist) rule.delete() @@ -137,23 +166,35 @@ def manage_list(request, email_or_name=None, acronym=None): for r in rules: r.matching_documents_count = docs_matching_community_list_rule(r).count() - empty_rule_forms = { rule_type: SearchRuleForm(clist, rule_type) for rule_type, _ in SearchRule.RULE_TYPES } + empty_rule_forms = { + rule_type: SearchRuleForm(clist, rule_type) + for rule_type, _ in SearchRule.RULE_TYPES + } total_count = docs_tracked_by_community_list(clist).count() - all_forms = [f for f in [rule_type_form, rule_form, add_doc_form, *empty_rule_forms.values()] - if f is not None] - return render(request, 'community/manage_list.html', { - 'clist': clist, - 'rules': rules, - 'individually_added': clist.added_docs.all() if clist.pk is not None else [], - 'rule_type_form': rule_type_form, - 'rule_form': rule_form, - 'empty_rule_forms': empty_rule_forms, - 'total_count': total_count, - 'add_doc_form': add_doc_form, - 'all_forms': all_forms, - }) + all_forms = [ + f + for f in [rule_type_form, rule_form, add_doc_form, *empty_rule_forms.values()] + if f is not None + ] + return render( + request, + "community/manage_list.html", + { + "clist": clist, + "rules": rules, + "individually_added": ( + clist.added_docs.all() if clist.pk is not None else [] + ), + "rule_type_form": rule_type_form, + "rule_form": rule_form, + "empty_rule_forms": empty_rule_forms, + "total_count": total_count, + "add_doc_form": add_doc_form, + "all_forms": all_forms, + }, + ) @login_required @@ -161,24 +202,33 @@ def track_document(request, name, email_or_name=None, acronym=None): doc = get_object_or_404(Document, name=name) if request.method == "POST": - clist = lookup_community_list(request, email_or_name, acronym) # may raise Http404 + clist = lookup_community_list( + request, email_or_name, acronym + ) # may raise Http404 if not can_manage_community_list(request.user, clist): permission_denied(request, "You do not have permission to access this view") if clist.pk is None: clist.save() - if not doc in clist.added_docs.all(): + if doc not in clist.added_docs.all(): clist.added_docs.add(doc) if is_ajax(request): - return HttpResponse(json.dumps({ 'success': True }), content_type='application/json') + return HttpResponse( + json.dumps({"success": True}), content_type="application/json" + ) else: return HttpResponseRedirect(clist.get_absolute_url()) - return render(request, "community/track_document.html", { - "name": doc.name, - }) + return render( + request, + "community/track_document.html", + { + "name": doc.name, + }, + ) + @login_required def untrack_document(request, name, email_or_name=None, acronym=None): @@ -192,28 +242,34 @@ def untrack_document(request, name, email_or_name=None, acronym=None): clist.added_docs.remove(doc) if is_ajax(request): - return HttpResponse(json.dumps({ 'success': True }), content_type='application/json') + return HttpResponse( + json.dumps({"success": True}), content_type="application/json" + ) else: return HttpResponseRedirect(clist.get_absolute_url()) - return render(request, "community/untrack_document.html", { - "name": doc.name, - }) + return render( + request, + "community/untrack_document.html", + { + "name": doc.name, + }, + ) @ignore_view_kwargs("group_type") def export_to_csv(request, email_or_name=None, acronym=None): clist = lookup_community_list(request, email_or_name, acronym) # may raise Http404 - response = HttpResponse(content_type='text/csv') + response = HttpResponse(content_type="text/csv") if clist.group: filename = "%s-draft-list.csv" % clist.group.acronym else: filename = "draft-list.csv" - response['Content-Disposition'] = 'attachment; filename=%s' % filename + response["Content-Disposition"] = "attachment; filename=%s" % filename - writer = csv.writer(response, dialect=csv.excel, delimiter=str(',')) + writer = csv.writer(response, dialect=csv.excel, delimiter=str(",")) header = [ "Name", @@ -226,12 +282,12 @@ def export_to_csv(request, email_or_name=None, acronym=None): ] writer.writerow(header) - docs = docs_tracked_by_community_list(clist).select_related('type', 'group', 'ad') + docs = docs_tracked_by_community_list(clist).select_related("type", "group", "ad") for doc in docs.prefetch_related("states", "tags"): row = [] row.append(doc.name) row.append(doc.title) - e = doc.latest_event(type='new_revision') + e = doc.latest_event(type="new_revision") row.append(e.time.strftime("%Y-%m-%d") if e else "") row.append(strip_tags(doc.friendly_state())) row.append(doc.group.acronym if doc.group else "") @@ -242,39 +298,54 @@ def export_to_csv(request, email_or_name=None, acronym=None): return response + @ignore_view_kwargs("group_type") def feed(request, email_or_name=None, acronym=None): clist = lookup_community_list(request, email_or_name, acronym) # may raise Http404 - significant = request.GET.get('significant', '') == '1' - - documents = docs_tracked_by_community_list(clist).values_list('pk', flat=True) - since = timezone.now() - datetime.timedelta(days=14) - - events = DocEvent.objects.filter( - doc__id__in=documents, - time__gte=since, - ).distinct().order_by('-time', '-id').select_related("doc") + significant = request.GET.get("significant", "") == "1" + + documents = docs_tracked_by_community_list(clist).values_list("pk", flat=True) + updated = timezone.now() + since = updated - datetime.timedelta(days=14) + + events = ( + DocEvent.objects.filter( + doc__id__in=documents, + time__gte=since, + ) + .distinct() + .order_by("-time", "-id") + .select_related("doc") + ) if significant: - events = events.filter(type="changed_state", statedocevent__state__in=list(states_of_significant_change())) + events = events.filter( + type="changed_state", + statedocevent__state__in=list(states_of_significant_change()), + ) host = request.get_host() - feed_url = 'https://%s%s' % (host, request.get_full_path()) + feed_url = "https://%s%s" % (host, request.get_full_path()) feed_id = uuid.uuid5(uuid.NAMESPACE_URL, str(feed_url)) - title = '%s RSS Feed' % clist.long_name() + title = "%s RSS Feed" % clist.long_name() if significant: - subtitle = 'Significant document changes' + subtitle = "Significant document changes" else: - subtitle = 'Document changes' - - return render(request, 'community/atom.xml', { - 'clist': clist, - 'entries': events[:50], - 'title': title, - 'subtitle': subtitle, - 'id': feed_id.urn, - 'updated': timezone.now(), - }, content_type='text/xml') + subtitle = "Document changes" + + return render( + request, + "community/atom.xml", + { + "clist": clist, + "entries": events[:50], + "title": title, + "subtitle": subtitle, + "id": feed_id.urn, + "updated": updated, + }, + content_type="text/xml", + ) @login_required @@ -286,9 +357,11 @@ def subscription(request, email_or_name=None, acronym=None): person = request.user.person - existing_subscriptions = EmailSubscription.objects.filter(community_list=clist, email__person=person) + existing_subscriptions = EmailSubscription.objects.filter( + community_list=clist, email__person=person + ) - if request.method == 'POST': + if request.method == "POST": action = request.POST.get("action") if action == "subscribe": form = SubscriptionForm(person, clist, request.POST) @@ -300,14 +373,20 @@ def subscription(request, email_or_name=None, acronym=None): return HttpResponseRedirect("") elif action == "unsubscribe": - existing_subscriptions.filter(pk=request.POST.get("subscription_id")).delete() + existing_subscriptions.filter( + pk=request.POST.get("subscription_id") + ).delete() return HttpResponseRedirect("") else: form = SubscriptionForm(person, clist) - return render(request, 'community/subscription.html', { - 'clist': clist, - 'form': form, - 'existing_subscriptions': existing_subscriptions, - }) + return render( + request, + "community/subscription.html", + { + "clist": clist, + "form": form, + "existing_subscriptions": existing_subscriptions, + }, + ) diff --git a/ietf/context_processors.py b/ietf/context_processors.py index baa8d7a5d2..5aaa4ab256 100644 --- a/ietf/context_processors.py +++ b/ietf/context_processors.py @@ -5,6 +5,7 @@ from django.conf import settings from django.utils import timezone from ietf import __version__, __patch__, __release_branch__, __release_hash__ +from opentelemetry.propagate import inject def server_mode(request): return {'server_mode': settings.SERVER_MODE} @@ -51,3 +52,8 @@ def timezone_now(request): return { 'timezone_now': timezone.now(), } + +def traceparent_id(request): + context_extras = {} + inject(context_extras) + return { "otel": context_extras } diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py index db3b24b2d2..86f5ac5fda 100644 --- a/ietf/doc/admin.py +++ b/ietf/doc/admin.py @@ -1,10 +1,12 @@ -# Copyright The IETF Trust 2010-2021, All Rights Reserved +# Copyright The IETF Trust 2010-2025, All Rights Reserved # -*- coding: utf-8 -*- from django.contrib import admin from django.db import models from django import forms +from django.db.models import QuerySet +from rangefilter.filters import DateRangeQuickSelectListFilterBuilder from .models import (StateType, State, RelatedDocument, DocumentAuthor, Document, RelatedDocHistory, DocHistoryAuthor, DocHistory, DocReminder, DocEvent, NewRevisionDocEvent, @@ -12,32 +14,37 @@ TelechatDocEvent, BallotPositionDocEvent, ReviewRequestDocEvent, InitialReviewDocEvent, AddedMessageEvent, SubmissionDocEvent, DeletedEvent, EditedAuthorsDocEvent, DocumentURL, ReviewAssignmentDocEvent, IanaExpertDocEvent, IRSGBallotDocEvent, DocExtResource, DocumentActionHolder, - BofreqEditorDocEvent, BofreqResponsibleDocEvent, StoredObject ) + BofreqEditorDocEvent, BofreqResponsibleDocEvent, StoredObject, RfcAuthor, + EditedRfcAuthorsDocEvent, RpcAssignmentDocEvent) +from ietf.utils.admin import SaferTabularInline from ietf.utils.validators import validate_external_resource_value +from .storage_utils import force_replication +from .utils import replicate_stored_objects_for_document + class StateTypeAdmin(admin.ModelAdmin): list_display = ["slug", "label"] admin.site.register(StateType, StateTypeAdmin) class StateAdmin(admin.ModelAdmin): - list_display = ["slug", "type", 'name', 'order', 'desc'] - list_filter = ["type", ] + list_display = ["slug", "type", 'name', 'order', 'desc', "used"] + list_filter = ["type", "used"] search_fields = ["slug", "type__label", "type__slug", "name", "desc"] filter_horizontal = ["next_states"] admin.site.register(State, StateAdmin) -class DocAuthorInline(admin.TabularInline): +class DocAuthorInline(SaferTabularInline): model = DocumentAuthor raw_id_fields = ['person', 'email'] extra = 1 -class DocActionHolderInline(admin.TabularInline): +class DocActionHolderInline(SaferTabularInline): model = DocumentActionHolder raw_id_fields = ['person'] extra = 1 -class RelatedDocumentInline(admin.TabularInline): +class RelatedDocumentInline(SaferTabularInline): model = RelatedDocument fk_name= 'source' def this(self, instance): @@ -47,7 +54,7 @@ def this(self, instance): raw_id_fields = ['target'] extra = 1 -class AdditionalUrlInLine(admin.TabularInline): +class AdditionalUrlInLine(SaferTabularInline): model = DocumentURL fields = ['tag','desc','url',] extra = 1 @@ -70,7 +77,9 @@ class DocumentAuthorAdmin(admin.ModelAdmin): search_fields = ['document__name', 'person__name', 'email__address', 'affiliation', 'country'] raw_id_fields = ["document", "person", "email"] admin.site.register(DocumentAuthor, DocumentAuthorAdmin) - + + + class DocumentAdmin(admin.ModelAdmin): list_display = ['name', 'rev', 'group', 'pages', 'intended_std_level', 'author_list', 'time'] search_fields = ['name'] @@ -78,6 +87,7 @@ class DocumentAdmin(admin.ModelAdmin): raw_id_fields = ['group', 'shepherd', 'ad'] inlines = [DocAuthorInline, DocActionHolderInline, RelatedDocumentInline, AdditionalUrlInLine] form = DocumentForm + actions = ["replicate_stored_objects"] def save_model(self, request, obj, form, change): e = DocEvent.objects.create( @@ -92,6 +102,22 @@ def save_model(self, request, obj, form, change): def state(self, instance): return self.get_state() + @admin.action(description="Replicate related blobs") + def replicate_stored_objects(self, request, queryset: QuerySet[Document]): + doc_count = 0 + stored_obj_count = 0 + for doc in queryset.all(): + doc_count += 1 + if isinstance(doc, Document): + stored_obj_count += replicate_stored_objects_for_document(doc) + self.message_user( + request, + ( + f"Queued replication of a total of {stored_obj_count} StoredObject(s) " + f"for {doc_count} Document(s)" + ) + ) + admin.site.register(Document, DocumentAdmin) class DocHistoryAdmin(admin.ModelAdmin): @@ -172,6 +198,7 @@ def short_desc(self, obj): admin.site.register(TelechatDocEvent, DocEventAdmin) admin.site.register(InitialReviewDocEvent, DocEventAdmin) admin.site.register(EditedAuthorsDocEvent, DocEventAdmin) +admin.site.register(EditedRfcAuthorsDocEvent, DocEventAdmin) admin.site.register(IanaExpertDocEvent, DocEventAdmin) class BallotPositionDocEventAdmin(DocEventAdmin): @@ -202,6 +229,10 @@ class SubmissionDocEventAdmin(DocEventAdmin): raw_id_fields = DocEventAdmin.raw_id_fields + ["submission"] admin.site.register(SubmissionDocEvent, SubmissionDocEventAdmin) +class RpcAssignmentDocEventAdmin(DocEventAdmin): + search_fields = DocEventAdmin.search_fields + ["assignments"] +admin.site.register(RpcAssignmentDocEvent, RpcAssignmentDocEventAdmin) + class DocumentUrlAdmin(admin.ModelAdmin): list_display = ['id', 'doc', 'tag', 'url', 'desc', ] search_fields = ['doc__name', 'url', ] @@ -220,7 +251,39 @@ class DocExtResourceAdmin(admin.ModelAdmin): admin.site.register(DocExtResource, DocExtResourceAdmin) class StoredObjectAdmin(admin.ModelAdmin): - list_display = ['store', 'name', 'modified', 'deleted'] - list_filter = ['deleted'] - search_fields = ['store', 'name', 'doc_name', 'doc_rev', 'deleted'] + list_display = ['store', 'name', 'doc_name', 'modified', 'is_deleted'] + list_filter = [ + 'store', + ('modified', DateRangeQuickSelectListFilterBuilder()), + ('deleted', DateRangeQuickSelectListFilterBuilder()), + ] + search_fields = ['name', 'doc_name', 'doc_rev'] + list_display_links = ['name'] + actions = ["replicate_stored_object"] + + @admin.display(boolean=True, description="Deleted?", ordering="deleted") + def is_deleted(self, instance): + return instance.deleted is not None + + @admin.action(description="Replicate related blobs") + def replicate_stored_object(self, request, queryset: QuerySet[StoredObject]): + stored_obj_count = 0 + for stored_object in queryset.all(): + if isinstance(stored_object, StoredObject): + force_replication(kind=stored_object.store, name=stored_object.name) + stored_obj_count += 1 + self.message_user( + request, + f"Queued replication of a total of {stored_obj_count} StoredObject(s)", + ) + + admin.site.register(StoredObject, StoredObjectAdmin) + +class RfcAuthorAdmin(admin.ModelAdmin): + # the email field in the list_display/readonly_fields works through a @property + list_display = ['id', 'document', 'titlepage_name', 'person', 'email', 'affiliation', 'country', 'order'] + search_fields = ['document__name', 'titlepage_name', 'person__name', 'person__email__address', 'affiliation', 'country'] + raw_id_fields = ["document", "person"] + readonly_fields = ["email"] +admin.site.register(RfcAuthor, RfcAuthorAdmin) diff --git a/ietf/doc/api.py b/ietf/doc/api.py new file mode 100644 index 0000000000..73fff6b27f --- /dev/null +++ b/ietf/doc/api.py @@ -0,0 +1,213 @@ +# Copyright The IETF Trust 2024-2026, All Rights Reserved +"""Doc API implementations""" + +from django.db.models import ( + BooleanField, + Count, + OuterRef, + Prefetch, + Q, + QuerySet, + Subquery, +) +from django.db.models.functions import TruncDate +from django_filters import rest_framework as filters +from rest_framework import filters as drf_filters +from rest_framework.mixins import ListModelMixin, RetrieveModelMixin +from rest_framework.pagination import LimitOffsetPagination +from rest_framework.viewsets import GenericViewSet + +from ietf.group.models import Group +from ietf.name.models import StreamName, DocTypeName +from ietf.utils.timezone import RPC_TZINFO +from .models import ( + Document, + DocEvent, + RelatedDocument, + DocumentAuthor, + SUBSERIES_DOC_TYPE_IDS, +) +from .serializers import ( + RfcMetadataSerializer, + RfcStatus, + RfcSerializer, + SubseriesDocSerializer, +) + + +class RfcLimitOffsetPagination(LimitOffsetPagination): + default_limit = 10 + max_limit = 500 + + +class NumberInFilter(filters.BaseInFilter, filters.NumberFilter): + """Filter against a comma-separated list of numbers""" + pass + + +class RfcFilter(filters.FilterSet): + published = filters.DateFromToRangeFilter() + stream = filters.ModelMultipleChoiceFilter( + queryset=StreamName.objects.filter(used=True) + ) + number = NumberInFilter( + field_name="rfc_number" + ) + group = filters.ModelMultipleChoiceFilter( + queryset=Group.objects.all(), + field_name="group__acronym", + to_field_name="acronym", + ) + area = filters.ModelMultipleChoiceFilter( + queryset=Group.objects.areas(), + field_name="group__parent__acronym", + to_field_name="acronym", + ) + status = filters.MultipleChoiceFilter( + choices=[(slug, slug) for slug in RfcStatus.status_slugs], + method=RfcStatus.filter, + ) + sort = filters.OrderingFilter( + fields=( + ("rfc_number", "number"), # ?sort=number / ?sort=-number + ("published", "published"), # ?sort=published / ?sort=-published + ), + ) + + +class PrefetchRelatedDocument(Prefetch): + """Prefetch via a RelatedDocument + + Prefetches following RelatedDocument relationships to other docs. By default, includes + those for which the current RFC is the `source`. If `reverse` is True, includes those + for which it is the `target` instead. Defaults to only "rfc" documents. + """ + + @staticmethod + def _get_queryset(relationship_id, reverse, doc_type_ids): + """Get queryset to use for the prefetch""" + if isinstance(doc_type_ids, str): + doc_type_ids = (doc_type_ids,) + + return RelatedDocument.objects.filter( + **{ + "relationship_id": relationship_id, + f"{'source' if reverse else 'target'}__type_id__in": doc_type_ids, + } + ).select_related("source" if reverse else "target") + + def __init__(self, to_attr, relationship_id, reverse=False, doc_type_ids="rfc"): + super().__init__( + lookup="targets_related" if reverse else "relateddocument_set", + queryset=self._get_queryset(relationship_id, reverse, doc_type_ids), + to_attr=to_attr, + ) + + +def augment_rfc_queryset(queryset: QuerySet[Document]): + return ( + queryset.select_related("std_level", "stream") + .prefetch_related( + Prefetch( + "group", + Group.objects.select_related("parent"), + ), + Prefetch( + "documentauthor_set", + DocumentAuthor.objects.select_related("email", "person"), + ), + PrefetchRelatedDocument( + to_attr="drafts", + relationship_id="became_rfc", + doc_type_ids="draft", + reverse=True, + ), + PrefetchRelatedDocument(to_attr="obsoletes", relationship_id="obs"), + PrefetchRelatedDocument( + to_attr="obsoleted_by", relationship_id="obs", reverse=True + ), + PrefetchRelatedDocument(to_attr="updates", relationship_id="updates"), + PrefetchRelatedDocument( + to_attr="updated_by", relationship_id="updates", reverse=True + ), + PrefetchRelatedDocument( + to_attr="subseries", + relationship_id="contains", + reverse=True, + doc_type_ids=SUBSERIES_DOC_TYPE_IDS, + ), + ) + .annotate( + published_datetime=Subquery( + DocEvent.objects.filter( + doc_id=OuterRef("pk"), + type="published_rfc", + ) + .order_by("-time") + .values("time")[:1] + ), + ) + .annotate(published=TruncDate("published_datetime", tzinfo=RPC_TZINFO)) + .annotate( + # Count of "verified-errata" tags will be 1 or 0, convert to Boolean + has_errata=Count( + "tags", + filter=Q( + tags__slug="verified-errata", + ), + output_field=BooleanField(), + ) + ) + ) + + +class RfcViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): + api_key_endpoint = "ietf.api.red_api" # matches prefix in ietf/api/urls.py + lookup_field = "rfc_number" + queryset = augment_rfc_queryset( + Document.objects.filter(type_id="rfc", rfc_number__isnull=False) + ).order_by("-rfc_number") + + pagination_class = RfcLimitOffsetPagination + filter_backends = [filters.DjangoFilterBackend, drf_filters.SearchFilter] + filterset_class = RfcFilter + search_fields = ["title", "abstract"] + + def get_serializer_class(self): + if self.action == "retrieve": + return RfcSerializer + return RfcMetadataSerializer + + +class PrefetchSubseriesContents(Prefetch): + def __init__(self, to_attr): + super().__init__( + lookup="relateddocument_set", + queryset=RelatedDocument.objects.filter( + relationship_id="contains", + target__type_id="rfc", + ).prefetch_related( + Prefetch( + "target", + queryset=augment_rfc_queryset(Document.objects.all()), + ) + ), + to_attr=to_attr, + ) + + +class SubseriesFilter(filters.FilterSet): + type = filters.ModelMultipleChoiceFilter( + queryset=DocTypeName.objects.filter(pk__in=SUBSERIES_DOC_TYPE_IDS) + ) + + +class SubseriesViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): + api_key_endpoint = "ietf.api.red_api" # matches prefix in ietf/api/urls.py + lookup_field = "name" + serializer_class = SubseriesDocSerializer + queryset = Document.objects.subseries_docs().prefetch_related( + PrefetchSubseriesContents(to_attr="contents") + ) + filter_backends = [filters.DjangoFilterBackend] + filterset_class = SubseriesFilter diff --git a/ietf/doc/expire.py b/ietf/doc/expire.py index bf8523aa98..d42af628f8 100644 --- a/ietf/doc/expire.py +++ b/ietf/doc/expire.py @@ -38,22 +38,46 @@ def expirable_drafts(queryset=None): # Populate this first time through (but after django has been set up) if nonexpirable_states is None: # all IESG states except I-D Exists and Dead block expiry - nonexpirable_states = list(State.objects.filter(used=True, type="draft-iesg").exclude(slug__in=("idexists", "dead"))) + nonexpirable_states = list( + State.objects.filter(used=True, type="draft-iesg").exclude( + slug__in=("idexists", "dead") + ) + ) # sent to RFC Editor and RFC Published block expiry (the latter # shouldn't be possible for an active draft, though) - nonexpirable_states += list(State.objects.filter(used=True, type__in=("draft-stream-iab", "draft-stream-irtf", "draft-stream-ise"), slug__in=("rfc-edit", "pub"))) + nonexpirable_states += list( + State.objects.filter( + used=True, + type__in=( + "draft-stream-iab", + "draft-stream-irtf", + "draft-stream-ise", + "draft-stream-editorial", + ), + slug__in=("rfc-edit", "pub"), + ) + ) # other IRTF states that block expiration - nonexpirable_states += list(State.objects.filter(used=True, type_id="draft-stream-irtf", slug__in=("irsgpoll", "iesg-rev",))) - - return queryset.filter( - states__type="draft", states__slug="active" - ).exclude( - expires=None - ).exclude( - states__in=nonexpirable_states - ).exclude( - tags="rfc-rev" # under review by the RFC Editor blocks expiry - ).distinct() + nonexpirable_states += list( + State.objects.filter( + used=True, + type_id="draft-stream-irtf", + slug__in=( + "irsgpoll", + "iesg-rev", + ), + ) + ) + + return ( + queryset.filter(states__type="draft", states__slug="active") + .exclude(expires=None) + .exclude(states__in=nonexpirable_states) + .exclude( + tags="rfc-rev" # under review by the RFC Editor blocks expiry + ) + .distinct() + ) def get_soon_to_expire_drafts(days_of_warning): diff --git a/ietf/doc/factories.py b/ietf/doc/factories.py index 19aa9ecc9c..1a178c6f31 100644 --- a/ietf/doc/factories.py +++ b/ietf/doc/factories.py @@ -14,7 +14,7 @@ from ietf.doc.models import ( Document, DocEvent, NewRevisionDocEvent, State, DocumentAuthor, StateDocEvent, BallotPositionDocEvent, BallotDocEvent, BallotType, IRSGBallotDocEvent, TelechatDocEvent, - DocumentActionHolder, BofreqEditorDocEvent, BofreqResponsibleDocEvent, DocExtResource ) + DocumentActionHolder, BofreqEditorDocEvent, BofreqResponsibleDocEvent, DocExtResource, RfcAuthor ) from ietf.group.models import Group from ietf.person.factories import PersonFactory from ietf.group.factories import RoleFactory @@ -311,6 +311,12 @@ class Meta: def desc(self): return 'New version available %s-%s'%(self.doc.name,self.rev) +class PublishedRfcDocEventFactory(DocEventFactory): + class Meta: + model = DocEvent + type = "published_rfc" + doc = factory.SubFactory(WgRfcFactory) + class StateDocEventFactory(DocEventFactory): class Meta: model = StateDocEvent @@ -382,6 +388,18 @@ class Meta: country = factory.Faker('country') order = factory.LazyAttribute(lambda o: o.document.documentauthor_set.count() + 1) +class RfcAuthorFactory(factory.django.DjangoModelFactory): + class Meta: + model = RfcAuthor + + document = factory.SubFactory(DocumentFactory) + titlepage_name = factory.LazyAttribute( + lambda obj: " ".join([obj.person.initials(), obj.person.last_name()]) + ) + person = factory.SubFactory('ietf.person.factories.PersonFactory') + affiliation = factory.Faker('company') + order = factory.LazyAttribute(lambda o: o.document.rfcauthor_set.count() + 1) + class WgDocumentAuthorFactory(DocumentAuthorFactory): document = factory.SubFactory(WgDraftFactory) diff --git a/ietf/doc/feeds.py b/ietf/doc/feeds.py index 500ed3cb18..7472b14c18 100644 --- a/ietf/doc/feeds.py +++ b/ietf/doc/feeds.py @@ -1,11 +1,11 @@ -# Copyright The IETF Trust 2007-2020, All Rights Reserved -# -*- coding: utf-8 -*- +# Copyright The IETF Trust 2007-2026, All Rights Reserved import debug # pyflakes:ignore import datetime import unicodedata +from django.conf import settings from django.contrib.syndication.views import Feed, FeedDoesNotExist from django.utils.feedgenerator import Atom1Feed, Rss201rev2Feed from django.urls import reverse as urlreverse @@ -224,7 +224,7 @@ def item_extra_kwargs(self, item): extra.update({"dcterms_accessRights": "gratis"}) extra.update({"dcterms_format": "text/html"}) media_contents = [] - if item.rfc_number < 8650: + if item.rfc_number < settings.FIRST_V3_RFC: if item.rfc_number not in [8, 9, 51, 418, 500, 530, 589]: for fmt, media_type in [("txt", "text/plain"), ("html", "text/html")]: media_contents.append( @@ -234,14 +234,6 @@ def item_extra_kwargs(self, item): "is_format_of": self.item_link(item), } ) - if item.rfc_number not in [571, 587]: - media_contents.append( - { - "url": f"https://www.rfc-editor.org/rfc/pdfrfc/{item.name}.txt.pdf", - "media_type": "application/pdf", - "is_format_of": self.item_link(item), - } - ) else: media_contents.append( { @@ -263,9 +255,11 @@ def item_extra_kwargs(self, item): ) extra.update({"media_contents": media_contents}) - extra.update({"doi": "10.17487/%s" % item.name.upper()}) extra.update( - {"doiuri": "http://dx.doi.org/10.17487/%s" % item.name.upper()} + { + "doi": item.doi, + "doiuri": f"https://doi.org/{item.doi}", + } ) # R104 Publisher (Mandatory - but we need a string from them first) diff --git a/ietf/doc/forms.py b/ietf/doc/forms.py index 8a1e9ecb98..768d6f96af 100644 --- a/ietf/doc/forms.py +++ b/ietf/doc/forms.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2013-2020, All Rights Reserved +# Copyright The IETF Trust 2013-2025, All Rights Reserved # -*- coding: utf-8 -*- @@ -9,7 +9,7 @@ from django.core.validators import validate_email from ietf.doc.fields import SearchableDocumentField, SearchableDocumentsField -from ietf.doc.models import RelatedDocument, DocExtResource +from ietf.doc.models import RelatedDocument, DocExtResource, State from ietf.iesg.models import TelechatDate from ietf.iesg.utils import telechat_page_count from ietf.person.fields import SearchablePersonField, SearchablePersonsField @@ -61,7 +61,7 @@ class DocAuthorChangeBasisForm(forms.Form): basis = forms.CharField(max_length=255, label='Reason for change', help_text='What is the source or reasoning for the changes to the author list?') - + class AdForm(forms.Form): ad = forms.ModelChoiceField(Person.objects.filter(role__name="ad", role__group__state="active", role__group__type='area').order_by('name'), label="Shepherding AD", empty_label="(None)", required=True) @@ -288,3 +288,10 @@ def clean_name_fragment(self): if any(c in name_fragment for c in disallowed_characters): raise ValidationError(f"The following characters are disallowed: {', '.join(disallowed_characters)}") return name_fragment + + +class ChangeStatementStateForm(forms.Form): + state = forms.ModelChoiceField( + State.objects.filter(used=True, type="statement"), + empty_label=None, + ) diff --git a/ietf/doc/management/commands/find_github_backup_info.py b/ietf/doc/management/commands/find_github_backup_info.py deleted file mode 100644 index f1f71452df..0000000000 --- a/ietf/doc/management/commands/find_github_backup_info.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright The IETF Trust 2020, All Rights Reserved - - -import github3 - -from collections import Counter -from urllib.parse import urlparse - -from django.conf import settings -from django.core.management.base import BaseCommand, CommandError - -from ietf.doc.models import DocExtResource -from ietf.group.models import GroupExtResource -from ietf.person.models import PersonExtResource - -# TODO: Think more about submodules. This currently will only take top level repos, with the assumption that the clone will include arguments to grab all the submodules. -# As a consequence, we might end up pulling more than we need (or that the org or user expected) -# Make sure this is what we want. - -class Command(BaseCommand): - help = ('Locate information about github repositories to backup') - - def add_arguments(self, parser): - parser.add_argument('--verbose', dest='verbose', action='store_true', help='Show counts of types of repositories') - - def handle(self, *args, **options): - - if not (hasattr(settings,'GITHUB_BACKUP_API_KEY') and settings.GITHUB_BACKUP_API_KEY): - raise CommandError("ERROR: can't find GITHUB_BACKUP_API_KEY") # TODO: at >= py3.1, use returncode - - github = github3.login(token = settings.GITHUB_BACKUP_API_KEY) - owners = dict() - repos = set() - - for cls in (DocExtResource, GroupExtResource, PersonExtResource): - for res in cls.objects.filter(name_id__in=('github_repo','github_org')): - path_parts = urlparse(res.value).path.strip('/').split('/') - if not path_parts or not path_parts[0]: - continue - - owner = path_parts[0] - - if owner not in owners: - try: - gh_owner = github.user(username=owner) - owners[owner] = gh_owner - except github3.exceptions.NotFoundError: - continue - - if gh_owner.type in ('User', 'Organization'): - if len(path_parts) > 1: - repo = path_parts[1] - if (owner, repo) not in repos: - try: - github.repository(owner,repo) - repos.add( (owner, repo) ) - except github3.exceptions.NotFoundError: - continue - else: - for repo in github.repositories_by(owner): - repos.add( (owner, repo.name) ) - - owner_types = Counter([owners[owner].type for owner in owners]) - if options['verbose']: - self.stdout.write("Owners:") - for key in owner_types: - self.stdout.write(" %s: %s"%(key,owner_types[key])) - self.stdout.write("Repositories: %d" % len(repos)) - for repo in sorted(repos): - self.stdout.write(" https://github.com/%s/%s" % repo ) - else: - for repo in sorted(repos): - self.stdout.write("%s/%s" % repo ) - diff --git a/ietf/doc/management/commands/reset_rfc_authors.py b/ietf/doc/management/commands/reset_rfc_authors.py deleted file mode 100644 index e2ab5f1208..0000000000 --- a/ietf/doc/management/commands/reset_rfc_authors.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright The IETF Trust 2024, All Rights Reserved - -# Reset an RFC's authors to those of the draft it came from -from django.core.management.base import BaseCommand, CommandError - -from ietf.doc.models import Document, DocEvent -from ietf.person.models import Person - - -class Command(BaseCommand): - def add_arguments(self, parser): - parser.add_argument("rfcnum", type=int, help="RFC number to modify") - parser.add_argument( - "--force", - action="store_true", - help="reset even if RFC already has authors", - ) - - def handle(self, *args, **options): - try: - rfc = Document.objects.get(type="rfc", rfc_number=options["rfcnum"]) - except Document.DoesNotExist: - raise CommandError( - f"rfc{options['rfcnum']} does not exist in the Datatracker." - ) - - draft = rfc.came_from_draft() - if draft is None: - raise CommandError(f"{rfc.name} did not come from a draft. Can't reset.") - - orig_authors = rfc.documentauthor_set.all() - if orig_authors.exists(): - # Potentially dangerous, so refuse unless "--force" is specified - if not options["force"]: - raise CommandError( - f"{rfc.name} already has authors. Not resetting. Use '--force' to reset anyway." - ) - removed_auth_names = list(orig_authors.values_list("person__name", flat=True)) - rfc.documentauthor_set.all().delete() - DocEvent.objects.create( - doc=rfc, - by=Person.objects.get(name="(System)"), - type="edited_authors", - desc=f"Removed all authors: {', '.join(removed_auth_names)}", - ) - self.stdout.write( - self.style.SUCCESS( - f"Removed author(s): {', '.join(removed_auth_names)}" - ) - ) - - for author in draft.documentauthor_set.all(): - # Copy the author but point at the new doc. - # See https://docs.djangoproject.com/en/4.2/topics/db/queries/#copying-model-instances - author.pk = None - author.id = None - author._state.adding = True - author.document = rfc - author.save() - self.stdout.write( - self.style.SUCCESS(f"Added author {author.person.name} <{author.email}>") - ) - auth_names = draft.documentauthor_set.values_list("person__name", flat=True) - DocEvent.objects.create( - doc=rfc, - by=Person.objects.get(name="(System)"), - type="edited_authors", - desc=f"Set authors from rev {draft.rev} of {draft.name}: {', '.join(auth_names)}", - ) diff --git a/ietf/doc/management/commands/tests.py b/ietf/doc/management/commands/tests.py deleted file mode 100644 index 8244d87266..0000000000 --- a/ietf/doc/management/commands/tests.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright The IETF Trust 2024, All Rights Reserved -# -*- coding: utf-8 -*- - -from io import StringIO - -from django.core.management import call_command, CommandError - -from ietf.doc.factories import DocumentAuthorFactory, WgDraftFactory, WgRfcFactory -from ietf.doc.models import Document, DocumentAuthor -from ietf.utils.test_utils import TestCase - - -class CommandTests(TestCase): - @staticmethod - def _call_command(command_name, *args, **options): - """Call command, capturing (and suppressing) output""" - out = StringIO() - err = StringIO() - options["stdout"] = out - options["stderr"] = err - call_command(command_name, *args, **options) - return out.getvalue(), err.getvalue() - - def test_reset_rfc_authors(self): - command_name = "reset_rfc_authors" - - draft = WgDraftFactory() - DocumentAuthorFactory.create_batch(3, document=draft) - rfc = WgRfcFactory() # rfc does not yet have a draft - DocumentAuthorFactory.create_batch(3, document=rfc) - bad_rfc_num = ( - 1 - + Document.objects.filter(rfc_number__isnull=False) - .order_by("-rfc_number") - .first() - .rfc_number - ) - docauthor_fields = [ - field.name - for field in DocumentAuthor._meta.get_fields() - if field.name not in ["document", "id"] - ] - - with self.assertRaises(CommandError, msg="Cannot reset a bad RFC number"): - self._call_command(command_name, bad_rfc_num) - - with self.assertRaises(CommandError, msg="Cannot reset an RFC with no draft"): - self._call_command(command_name, rfc.rfc_number) - - with self.assertRaises(CommandError, msg="Cannot force-reset an RFC with no draft"): - self._call_command(command_name, rfc.rfc_number, "--force") - - # Link the draft to the rfc - rfc.targets_related.create(relationship_id="became_rfc", source=draft) - - with self.assertRaises(CommandError, msg="Cannot reset an RFC with authors"): - self._call_command(command_name, rfc.rfc_number) - - # Calling with force should work - self._call_command(command_name, rfc.rfc_number, "--force") - self.assertCountEqual( - draft.documentauthor_set.values(*docauthor_fields), - rfc.documentauthor_set.values(*docauthor_fields), - ) - - # Calling on an RFC with no authors should also work - rfc.documentauthor_set.all().delete() - self._call_command(command_name, rfc.rfc_number) - self.assertCountEqual( - draft.documentauthor_set.values(*docauthor_fields), - rfc.documentauthor_set.values(*docauthor_fields), - ) diff --git a/ietf/doc/migrations/0026_change_wg_state_descriptions.py b/ietf/doc/migrations/0026_change_wg_state_descriptions.py new file mode 100644 index 0000000000..b02b12c97e --- /dev/null +++ b/ietf/doc/migrations/0026_change_wg_state_descriptions.py @@ -0,0 +1,117 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations + +def forward(apps, schema_editor): + State = apps.get_model("doc","State") + for name, desc in [ + ("WG Document","The document has been adopted by the Working Group (WG) and is under development. A document can only be adopted by one WG at a time. However, a document may be transferred between WGs."), + ("Parked WG Document","The Working Group (WG) document is in a temporary state where it will not be actively developed. The reason for the pause is explained via a datatracker comments section."), + ("Dead WG Document","The Working Group (WG) document has been abandoned by the WG. No further development is planned in this WG. A decision to resume work on this document and move it out of this state is possible."), + ("In WG Last Call","The Working Group (WG) document is currently subject to an active WG Last Call (WGLC) review per Section 7.4 of RFC2418."), + ("Waiting for Implementation","The progression of this Working Group (WG) document towards publication is paused as it awaits implementation. The process governing the approach to implementations is WG-specific."), + ("Held by WG","Held by Working Group (WG) chairs for administrative reasons. See document history for details."), + ("Waiting for WG Chair Go-Ahead","The Working Group (WG) document has completed Working Group Last Call (WGLC), but the WG chair(s) are not yet ready to call consensus on the document. The reasons for this may include comments from the WGLC need to be responded to, or a revision to the document is needed"), + ("WG Consensus: Waiting for Write-Up","The Working Group (WG) document has consensus to proceed to publication. However, the document is waiting for a document shepherd write-up per RFC4858."), + ("Submitted to IESG for Publication","The Working Group (WG) document has left the WG and been submitted to the Internet Engineering Steering Group (IESG) for evaluation and publication. See the “IESG State” or “RFC Editor State” for further details on the state of the document."), + ("Candidate for WG Adoption","The individual submission document has been marked by the Working Group (WG) chairs as a candidate for adoption by the WG, but no adoption call has been started."), + ("Call For Adoption By WG Issued","A call for adoption of the individual submission document has been issued by the Working Group (WG) chairs. This call is still running but the WG has not yet reached consensus for adoption."), + ("Adopted by a WG","The individual submission document has been adopted by the Working Group (WG), but a WG document replacing this document with the typical naming convention of 'draft- ietf-wgname-topic-nn' has not yet been submitted."), + ("Adopted for WG Info Only","The document is adopted by the Working Group (WG) for its internal use. The WG has decided that it will not pursue publication of it as an RFC."), + ]: + State.objects.filter(name=name).update(desc=desc) + +def reverse(apps, schema_editor): + State = apps.get_model("doc","State") + for name, desc in [ + ("WG Document","""4.2.4. WG Document + + The "WG Document" state describes an I-D that has been adopted by an IETF WG and is being actively developed. + + A WG Chair may transition an I-D into the "WG Document" state at any time as long as the I-D is not being considered or developed in any other WG. + + Alternatively, WG Chairs may rely upon new functionality to be added to the Datatracker to automatically move version-00 drafts into the "WG Document" state as described in Section 4.1. + + Under normal conditions, it should not be possible for an I-D to be in the "WG Document" state in more than one WG at a time. This said, I-Ds may be transferred from one WG to another with the consent of the WG Chairs and the responsible ADs."""), + ("Parked WG Document","""4.2.5. Parked WG Document + + A "Parked WG Document" is an I-D that has lost its author or editor, is waiting for another document to be written or for a review to be completed, or cannot be progressed by the working group for some other reason. + + Some of the annotation tags described in Section 4.3 may be used in conjunction with this state to indicate why an I-D has been parked, and/or what may need to happen for the I-D to be un-parked. + + Parking a WG draft will not prevent it from expiring; however, this state can be used to indicate why the I-D has stopped progressing in the WG. + + A "Parked WG Document" that is not expired may be transferred from one WG to another with the consent of the WG Chairs and the responsible ADs."""), + ("Dead WG Document","""4.2.6. Dead WG Document + + A "Dead WG Document" is an I-D that has been abandoned. Note that 'Dead' is not always a final state for a WG I-D. If consensus is subsequently achieved, a "Dead WG Document" may be resurrected. A "Dead WG Document" that is not resurrected will eventually expire. + + Note that an I-D that is declared to be "Dead" in one WG and that is not expired may be transferred to a non-dead state in another WG with the consent of the WG Chairs and the responsible ADs."""), + ("In WG Last Call","""4.2.7. In WG Last Call + + A document "In WG Last Call" is an I-D for which a WG Last Call (WGLC) has been issued and is in progress. + + Note that conducting a WGLC is an optional part of the IETF WG process, per Section 7.4 of RFC 2418 [RFC2418]. + + If a WG Chair decides to conduct a WGLC on an I-D, the "In WG Last Call" state can be used to track the progress of the WGLC. The Chair may configure the Datatracker to send a WGLC message to one or more mailing lists when the Chair moves the I-D into this state. The WG Chair may also be able to select a different set of mailing lists for a different document undergoing a WGLC; some documents may deserve coordination with other WGs. + + A WG I-D in this state should remain "In WG Last Call" until the WG Chair moves it to another state. The WG Chair may configure the Datatracker to send an e-mail after a specified period of time to remind or 'nudge' the Chair to conclude the WGLC and to determine the next state for the document. + + It is possible for one WGLC to lead into another WGLC for the same document. For example, an I-D that completed a WGLC as an "Informational" document may need another WGLC if a decision is taken to convert the I-D into a Standards Track document."""), + ("Waiting for Implementation","""In some areas, it can be desirable to wait for multiple interoperable implementations before progressing a draft to be an RFC, and in some WGs this is required. This state should be entered after WG Last Call has completed."""), + ("Held by WG","""Held by WG, see document history for details."""), + ("Waiting for WG Chair Go-Ahead","""4.2.8. Waiting for WG Chair Go-Ahead + + A WG Chair may wish to place an I-D that receives a lot of comments during a WGLC into the "Waiting for WG Chair Go-Ahead" state. This state describes an I-D that has undergone a WGLC; however, the Chair is not yet ready to call consensus on the document. + + If comments from the WGLC need to be responded to, or a revision to the I-D is needed, the Chair may place an I-D into this state until all of the WGLC comments are adequately addressed and the (possibly revised) document is in the I-D repository."""), + ("WG Consensus: Waiting for Write-Up","""4.2.9. WG Consensus: Waiting for Writeup + + A document in the "WG Consensus: Waiting for Writeup" state has essentially completed its development within the working group, and is nearly ready to be sent to the IESG for publication. The last thing to be done is the preparation of a protocol writeup by a Document Shepherd. The IESG requires that a document shepherd writeup be completed before publication of the I-D is requested. The IETF document shepherding process and the role of a WG Document Shepherd is described in RFC 4858 [RFC4858] + + A WG Chair may call consensus on an I-D without a formal WGLC and transition an I-D that was in the "WG Document" state directly into this state. + + The name of this state includes the words "Waiting for Writeup" because a good document shepherd writeup takes time to prepare."""), + ("Submitted to IESG for Publication","""4.2.10. Submitted to IESG for Publication + + This state describes a WG document that has been submitted to the IESG for publication and that has not been sent back to the working group for revision. + + An I-D in this state may be under review by the IESG, it may have been approved and be in the RFC Editor's queue, or it may have been published as an RFC. Other possibilities exist too. The document may be "Dead" (in the IESG state machine) or in a "Do Not Publish" state."""), + ("Candidate for WG Adoption","""The document has been marked as a candidate for WG adoption by the WG Chair. This state can be used before a call for adoption is issued (and the document is put in the "Call For Adoption By WG Issued" state), to indicate that the document is in the queue for a call for adoption, even if none has been issued yet."""), + ("Call For Adoption By WG Issued","""4.2.1. Call for Adoption by WG Issued + + The "Call for Adoption by WG Issued" state should be used to indicate when an I-D is being considered for adoption by an IETF WG. An I-D that is in this state is actively being considered for adoption and has not yet achieved consensus, preference, or selection in the WG. + + This state may be used to describe an I-D that someone has asked a WG to consider for adoption, if the WG Chair has agreed with the request. This state may also be used to identify an I-D that a WG Chair asked an author to write specifically for consideration as a candidate WG item [WGDTSPEC], and/or an I-D that is listed as a 'candidate draft' in the WG's charter. + + Under normal conditions, it should not be possible for an I-D to be in the "Call for Adoption by WG Issued" state in more than one working group at the same time. This said, it is not uncommon for authors to "shop" their I-Ds to more than one WG at a time, with the hope of getting their documents adopted somewhere. + + After this state is implemented in the Datatracker, an I-D that is in the "Call for Adoption by WG Issued" state will not be able to be "shopped" to any other WG without the consent of the WG Chairs and the responsible ADs impacted by the shopping. + + Note that Figure 1 includes an arc leading from this state to outside of the WG state machine. This illustrates that some I-Ds that are considered do not get adopted as WG drafts. An I-D that is not adopted as a WG draft will transition out of the WG state machine and revert back to having no stream-specific state; however, the status change history log of the I-D will record that the I-D was previously in the "Call for Adoption by WG Issued" state."""), + ("Adopted by a WG","""4.2.2. Adopted by a WG + + The "Adopted by a WG" state describes an individual submission I-D that an IETF WG has agreed to adopt as one of its WG drafts. + + WG Chairs who use this state will be able to clearly indicate when their WGs adopt individual submission I-Ds. This will facilitate the Datatracker's ability to correctly capture "Replaces" information for WG drafts and correct "Replaced by" information for individual submission I-Ds that have been replaced by WG drafts. + + This state is needed because the Datatracker uses the filename of an I-D as a key to search its database for status information about the I-D, and because the filename of a WG I-D is supposed to be different from the filename of an individual submission I-D. The filename of an individual submission I-D will typically be formatted as 'draft-author-wgname-topic-nn'. + + The filename of a WG document is supposed to be formatted as 'draft- ietf-wgname-topic-nn'. + + An individual I-D that is adopted by a WG may take weeks or months to be resubmitted by the author as a new (version-00) WG draft. If the "Adopted by a WG" state is not used, the Datatracker has no way to determine that an I-D has been adopted until a new version of the I-D is submitted to the WG by the author and until the I-D is approved for posting by a WG Chair."""), + ("Adopted for WG Info Only","""4.2.3. Adopted for WG Info Only + + The "Adopted for WG Info Only" state describes a document that contains useful information for the WG that adopted it, but the document is not intended to be published as an RFC. The WG will not actively develop the contents of the I-D or progress it for publication as an RFC. The only purpose of the I-D is to provide information for internal use by the WG."""), + ]: + State.objects.filter(name=name).update(desc=desc) + +class Migration(migrations.Migration): + + dependencies = [ + ("doc", "0025_storedobject_storedobject_unique_name_per_store"), + ] + + operations = [ + migrations.RunPython(forward, reverse) + ] diff --git a/ietf/doc/migrations/0027_alter_dochistory_title_alter_document_title.py b/ietf/doc/migrations/0027_alter_dochistory_title_alter_document_title.py new file mode 100644 index 0000000000..e0d8560e6f --- /dev/null +++ b/ietf/doc/migrations/0027_alter_dochistory_title_alter_document_title.py @@ -0,0 +1,41 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0026_change_wg_state_descriptions"), + ] + + operations = [ + migrations.AlterField( + model_name="dochistory", + name="title", + field=models.CharField( + max_length=255, + validators=[ + django.core.validators.ProhibitNullCharactersValidator, # type:ignore + django.core.validators.RegexValidator( + message="Please enter a string without control characters.", + regex="^[^\x01-\x1f]*$", + ), + ], + ), + ), + migrations.AlterField( + model_name="document", + name="title", + field=models.CharField( + max_length=255, + validators=[ + django.core.validators.ProhibitNullCharactersValidator, # type:ignore + django.core.validators.RegexValidator( + message="Please enter a string without control characters.", + regex="^[^\x01-\x1f]*$", + ), + ], + ), + ), + ] diff --git a/ietf/doc/migrations/0028_rfcauthor.py b/ietf/doc/migrations/0028_rfcauthor.py new file mode 100644 index 0000000000..776dc22eb1 --- /dev/null +++ b/ietf/doc/migrations/0028_rfcauthor.py @@ -0,0 +1,84 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations, models +import django.db.models.deletion +import ietf.utils.models + + +class Migration(migrations.Migration): + dependencies = [ + ("person", "0005_alter_historicalperson_pronouns_selectable_and_more"), + ("doc", "0027_alter_dochistory_title_alter_document_title"), + ] + + operations = [ + migrations.CreateModel( + name="RfcAuthor", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("titlepage_name", models.CharField(max_length=128)), + ("is_editor", models.BooleanField(default=False)), + ( + "affiliation", + models.CharField( + blank=True, + help_text="Organization/company used by author for submission", + max_length=100, + ), + ), + ( + "country", + models.CharField( + blank=True, + help_text="Country used by author for submission", + max_length=255, + ), + ), + ("order", models.IntegerField(default=1)), + ( + "document", + ietf.utils.models.ForeignKey( + limit_choices_to={"type_id": "rfc"}, + on_delete=django.db.models.deletion.CASCADE, + to="doc.document", + ), + ), + ( + "email", + ietf.utils.models.ForeignKey( + blank=True, + help_text="Email address used by author for submission", + null=True, + on_delete=django.db.models.deletion.PROTECT, + to="person.email", + ), + ), + ( + "person", + ietf.utils.models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.PROTECT, + to="person.person", + ), + ), + ], + options={ + "ordering": ["document", "order"], + "indexes": [ + models.Index( + fields=["document", "order"], + name="doc_rfcauth_documen_6b5dc4_idx", + ) + ], + }, + ), + ] diff --git a/ietf/doc/migrations/0029_editedrfcauthorsdocevent.py b/ietf/doc/migrations/0029_editedrfcauthorsdocevent.py new file mode 100644 index 0000000000..60837c5cb2 --- /dev/null +++ b/ietf/doc/migrations/0029_editedrfcauthorsdocevent.py @@ -0,0 +1,30 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0028_rfcauthor"), + ] + + operations = [ + migrations.CreateModel( + name="EditedRfcAuthorsDocEvent", + fields=[ + ( + "docevent_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="doc.docevent", + ), + ), + ], + bases=("doc.docevent",), + ), + ] diff --git a/ietf/doc/migrations/0030_alter_dochistory_title_alter_document_title.py b/ietf/doc/migrations/0030_alter_dochistory_title_alter_document_title.py new file mode 100644 index 0000000000..9ee858b2e8 --- /dev/null +++ b/ietf/doc/migrations/0030_alter_dochistory_title_alter_document_title.py @@ -0,0 +1,41 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0029_editedrfcauthorsdocevent"), + ] + + operations = [ + migrations.AlterField( + model_name="dochistory", + name="title", + field=models.CharField( + max_length=255, + validators=[ + django.core.validators.ProhibitNullCharactersValidator(), + django.core.validators.RegexValidator( + message="Please enter a string without control characters.", + regex="^[^\x01-\x1f]*$", + ), + ], + ), + ), + migrations.AlterField( + model_name="document", + name="title", + field=models.CharField( + max_length=255, + validators=[ + django.core.validators.ProhibitNullCharactersValidator(), + django.core.validators.RegexValidator( + message="Please enter a string without control characters.", + regex="^[^\x01-\x1f]*$", + ), + ], + ), + ), + ] diff --git a/ietf/doc/migrations/0031_change_draft_stream_ietf_state_descriptions.py b/ietf/doc/migrations/0031_change_draft_stream_ietf_state_descriptions.py new file mode 100644 index 0000000000..c664126da3 --- /dev/null +++ b/ietf/doc/migrations/0031_change_draft_stream_ietf_state_descriptions.py @@ -0,0 +1,57 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations + + +def forward(apps, schema_editor): + State = apps.get_model("doc", "State") + for name, desc in [ + ( + "Adopted by a WG", + "The individual submission document has been adopted by the Working Group (WG), but some administrative matter still needs to be completed (e.g., a WG document replacing this document with the typical naming convention of 'draft-ietf-wgname-topic-nn' has not yet been submitted).", + ), + ( + "WG Document", + "The document has been identified as a Working Group (WG) document and is under development per Section 7.2 of RFC2418.", + ), + ( + "Waiting for WG Chair Go-Ahead", + "The Working Group (WG) document has completed Working Group Last Call (WGLC), but the WG chairs are not yet ready to call consensus on the document. The reasons for this may include comments from the WGLC need to be responded to, or a revision to the document is needed.", + ), + ( + "Submitted to IESG for Publication", + "The Working Group (WG) document has been submitted to the Internet Engineering Steering Group (IESG) for evaluation and publication per Section 7.4 of RFC2418. See the “IESG State” or “RFC Editor State” for further details on the state of the document.", + ), + ]: + State.objects.filter(name=name).update(desc=desc, type="draft-stream-ietf") + + +def reverse(apps, schema_editor): + State = apps.get_model("doc", "State") + for name, desc in [ + ( + "Adopted by a WG", + "The individual submission document has been adopted by the Working Group (WG), but a WG document replacing this document with the typical naming convention of 'draft- ietf-wgname-topic-nn' has not yet been submitted.", + ), + ( + "WG Document", + "The document has been adopted by the Working Group (WG) and is under development. A document can only be adopted by one WG at a time. However, a document may be transferred between WGs.", + ), + ( + "Waiting for WG Chair Go-Ahead", + "The Working Group (WG) document has completed Working Group Last Call (WGLC), but the WG chair(s) are not yet ready to call consensus on the document. The reasons for this may include comments from the WGLC need to be responded to, or a revision to the document is needed", + ), + ( + "Submitted to IESG for Publication", + "The Working Group (WG) document has left the WG and been submitted to the Internet Engineering Steering Group (IESG) for evaluation and publication. See the “IESG State” or “RFC Editor State” for further details on the state of the document.", + ), + ]: + State.objects.filter(name=name).update(desc=desc, type="draft-stream-ietf") + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0030_alter_dochistory_title_alter_document_title"), + ] + + operations = [migrations.RunPython(forward, reverse)] diff --git a/ietf/doc/migrations/0032_remove_rfcauthor_email.py b/ietf/doc/migrations/0032_remove_rfcauthor_email.py new file mode 100644 index 0000000000..a0e147da59 --- /dev/null +++ b/ietf/doc/migrations/0032_remove_rfcauthor_email.py @@ -0,0 +1,16 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0031_change_draft_stream_ietf_state_descriptions"), + ] + + operations = [ + migrations.RemoveField( + model_name="rfcauthor", + name="email", + ), + ] diff --git a/ietf/doc/migrations/0033_dochistory_keywords_document_keywords.py b/ietf/doc/migrations/0033_dochistory_keywords_document_keywords.py new file mode 100644 index 0000000000..5e2513e15a --- /dev/null +++ b/ietf/doc/migrations/0033_dochistory_keywords_document_keywords.py @@ -0,0 +1,31 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations, models +import ietf.doc.models + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0032_remove_rfcauthor_email"), + ] + + operations = [ + migrations.AddField( + model_name="dochistory", + name="keywords", + field=models.JSONField( + default=list, + max_length=1000, + validators=[ietf.doc.models.validate_doc_keywords], + ), + ), + migrations.AddField( + model_name="document", + name="keywords", + field=models.JSONField( + default=list, + max_length=1000, + validators=[ietf.doc.models.validate_doc_keywords], + ), + ), + ] diff --git a/ietf/doc/migrations/0034_alter_dochistory_keywords_alter_document_keywords.py b/ietf/doc/migrations/0034_alter_dochistory_keywords_alter_document_keywords.py new file mode 100644 index 0000000000..2b89b67e88 --- /dev/null +++ b/ietf/doc/migrations/0034_alter_dochistory_keywords_alter_document_keywords.py @@ -0,0 +1,33 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations, models +import ietf.doc.models + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0033_dochistory_keywords_document_keywords"), + ] + + operations = [ + migrations.AlterField( + model_name="dochistory", + name="keywords", + field=models.JSONField( + blank=True, + default=list, + max_length=1000, + validators=[ietf.doc.models.validate_doc_keywords], + ), + ), + migrations.AlterField( + model_name="document", + name="keywords", + field=models.JSONField( + blank=True, + default=list, + max_length=1000, + validators=[ietf.doc.models.validate_doc_keywords], + ), + ), + ] diff --git a/ietf/doc/migrations/0035_add_rpc_queue_draft_rfceditor_states.py b/ietf/doc/migrations/0035_add_rpc_queue_draft_rfceditor_states.py new file mode 100644 index 0000000000..9805970ef0 --- /dev/null +++ b/ietf/doc/migrations/0035_add_rpc_queue_draft_rfceditor_states.py @@ -0,0 +1,31 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations + + +def forward(apps, schema_editor): + State = apps.get_model("doc", "State") + for slug, name in [("in_progress", "In Progress"), ("blocked", "Blocked")]: + State.objects.get_or_create( + type_id="draft-rfceditor", + slug=slug, + defaults={"name": name, "used": True, "desc": "", "order": 0}, + ) + + +def reverse(apps, schema_editor): + State = apps.get_model("doc", "State") + Document = apps.get_model("doc", "Document") + for slug in ("in_progress", "blocked"): + assert not Document.objects.filter( + states__type="draft-rfceditor", states__slug=slug + ).exists() + State.objects.filter(type_id="draft-rfceditor", slug=slug).delete() + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0034_alter_dochistory_keywords_alter_document_keywords"), + ] + + operations = [migrations.RunPython(forward, reverse)] diff --git a/ietf/doc/migrations/0036_alter_docevent_type.py b/ietf/doc/migrations/0036_alter_docevent_type.py new file mode 100644 index 0000000000..1cc11d4ee9 --- /dev/null +++ b/ietf/doc/migrations/0036_alter_docevent_type.py @@ -0,0 +1,92 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0035_add_rpc_queue_draft_rfceditor_states"), + ] + + operations = [ + migrations.AlterField( + model_name="docevent", + name="type", + field=models.CharField( + choices=[ + ("new_revision", "Added new revision"), + ("new_submission", "Uploaded new revision"), + ("changed_document", "Changed document metadata"), + ("added_comment", "Added comment"), + ("added_message", "Added message"), + ("edited_authors", "Edited the documents author list"), + ("deleted", "Deleted document"), + ("changed_state", "Changed state"), + ("changed_stream", "Changed document stream"), + ("expired_document", "Expired document"), + ("extended_expiry", "Extended expiry of document"), + ("requested_resurrect", "Requested resurrect"), + ("completed_resurrect", "Completed resurrect"), + ("changed_consensus", "Changed consensus"), + ("published_rfc", "Published RFC"), + ( + "added_suggested_replaces", + "Added suggested replacement relationships", + ), + ( + "reviewed_suggested_replaces", + "Reviewed suggested replacement relationships", + ), + ("changed_action_holders", "Changed action holders for document"), + ("changed_group", "Changed group"), + ("changed_protocol_writeup", "Changed protocol writeup"), + ("changed_charter_milestone", "Changed charter milestone"), + ("initial_review", "Set initial review time"), + ("changed_review_announcement", "Changed WG Review text"), + ("changed_action_announcement", "Changed WG Action text"), + ("started_iesg_process", "Started IESG process on document"), + ("created_ballot", "Created ballot"), + ("closed_ballot", "Closed ballot"), + ("sent_ballot_announcement", "Sent ballot announcement"), + ("changed_ballot_position", "Changed ballot position"), + ("changed_ballot_approval_text", "Changed ballot approval text"), + ("changed_ballot_writeup_text", "Changed ballot writeup text"), + ("changed_rfc_editor_note_text", "Changed RFC Editor Note text"), + ("changed_last_call_text", "Changed last call text"), + ("requested_last_call", "Requested last call"), + ("sent_last_call", "Sent last call"), + ("scheduled_for_telechat", "Scheduled for telechat"), + ("iesg_approved", "IESG approved document (no problem)"), + ("iesg_disapproved", "IESG disapproved document (do not publish)"), + ("approved_in_minute", "Approved in minute"), + ("iana_review", "IANA review comment"), + ("rfc_in_iana_registry", "RFC is in IANA registry"), + ( + "rfc_editor_received_announcement", + "Announcement was received by RFC Editor", + ), + ("requested_publication", "Publication at RFC Editor requested"), + ( + "sync_from_rfc_editor", + "Received updated information from RFC Editor", + ), + ("changed_rpc_assignments", "Changed RPC queue assignments"), + ("requested_review", "Requested review"), + ("assigned_review_request", "Assigned review request"), + ("closed_review_request", "Closed review request"), + ("closed_review_assignment", "Closed review assignment"), + ("downref_approved", "Downref approved"), + ("posted_related_ipr", "Posted related IPR"), + ("removed_related_ipr", "Removed related IPR"), + ( + "removed_objfalse_related_ipr", + "Removed Objectively False related IPR", + ), + ("changed_editors", "Changed BOF Request editors"), + ("published_statement", "Published statement"), + ("approved_slides", "Slides approved"), + ], + max_length=50, + ), + ), + ] diff --git a/ietf/doc/migrations/0037_rpcassignmentdocevent.py b/ietf/doc/migrations/0037_rpcassignmentdocevent.py new file mode 100644 index 0000000000..648376e118 --- /dev/null +++ b/ietf/doc/migrations/0037_rpcassignmentdocevent.py @@ -0,0 +1,31 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0036_alter_docevent_type"), + ] + + operations = [ + migrations.CreateModel( + name="RpcAssignmentDocEvent", + fields=[ + ( + "docevent_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="doc.docevent", + ), + ), + ("assignments", models.TextField(blank=True)), + ], + bases=("doc.docevent",), + ), + ] diff --git a/ietf/doc/models.py b/ietf/doc/models.py index 55da70972c..156bac4e77 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -1,7 +1,8 @@ -# Copyright The IETF Trust 2010-2023, All Rights Reserved +# Copyright The IETF Trust 2010-2026, All Rights Reserved # -*- coding: utf-8 -*- +from collections import namedtuple import datetime import logging import os @@ -11,6 +12,9 @@ from io import BufferedReader from pathlib import Path + +from django.core.exceptions import ValidationError +from django.db.models import Q from lxml import etree from typing import Optional, Protocol, TYPE_CHECKING, Union from weasyprint import HTML as wpHTML @@ -20,7 +24,11 @@ from django.core import checks from django.core.files.base import File from django.core.cache import caches -from django.core.validators import URLValidator, RegexValidator +from django.core.validators import ( + URLValidator, + RegexValidator, + ProhibitNullCharactersValidator, +) from django.urls import reverse as urlreverse from django.contrib.contenttypes.models import ContentType from django.conf import settings @@ -44,6 +52,7 @@ from ietf.person.utils import get_active_balloters from ietf.utils import log from ietf.utils.decorators import memoize +from ietf.utils.text import decode_document_content from ietf.utils.validators import validate_no_control_chars from ietf.utils.mail import formataddr from ietf.utils.models import ForeignKey @@ -102,12 +111,27 @@ class Meta: IESG_STATCHG_CONFLREV_ACTIVE_STATES = ("iesgeval", "defer") IESG_SUBSTATE_TAGS = ('ad-f-up', 'need-rev', 'extpty') + +def validate_doc_keywords(value): + if ( + not isinstance(value, list | tuple | set) + or not all(isinstance(elt, str) for elt in value) + ): + raise ValidationError("Value must be an array of strings") + + class DocumentInfo(models.Model): """Any kind of document. Draft, RFC, Charter, IPR Statement, Liaison Statement""" time = models.DateTimeField(default=timezone.now) # should probably have auto_now=True type = ForeignKey(DocTypeName, blank=True, null=True) # Draft, Agenda, Minutes, Charter, Discuss, Guideline, Email, Review, Issue, Wiki, External ... - title = models.CharField(max_length=255, validators=[validate_no_control_chars, ]) + title = models.CharField( + max_length=255, + validators=[ + ProhibitNullCharactersValidator(), + validate_no_control_chars, + ], + ) states = models.ManyToManyField(State, blank=True) # plain state (Active/Expired/...), IESG state, stream state tags = models.ManyToManyField(DocTagName, blank=True) # Revised ID Needed, ExternalParty, AD Followup, ... @@ -129,6 +153,18 @@ class DocumentInfo(models.Model): uploaded_filename = models.TextField(blank=True) note = models.TextField(blank=True) rfc_number = models.PositiveIntegerField(blank=True, null=True) # only valid for type="rfc" + keywords = models.JSONField( + default=list, + max_length=1000, + validators=[validate_doc_keywords], + blank=True, + ) + + @property + def doi(self) -> str | None: + if self.type_id == "rfc" and self.rfc_number is not None: + return f"{settings.IETF_DOI_PREFIX}/RFC{self.rfc_number:04d}" + return None def file_extension(self): if not hasattr(self, '_cached_extension'): @@ -228,14 +264,14 @@ def revisions_by_newrevisionevent(self): return revisions def get_href(self, meeting=None): - return self._get_ref(meeting=meeting,meeting_doc_refs=settings.MEETING_DOC_HREFS) + return self._get_ref(meeting=meeting, versioned=True) def get_versionless_href(self, meeting=None): - return self._get_ref(meeting=meeting,meeting_doc_refs=settings.MEETING_DOC_GREFS) + return self._get_ref(meeting=meeting, versioned=False) - def _get_ref(self, meeting=None, meeting_doc_refs=settings.MEETING_DOC_HREFS): + def _get_ref(self, meeting=None, versioned=True): """ Returns an url to the document text. This differs from .get_absolute_url(), which returns an url to the datatracker page for the document. @@ -244,12 +280,16 @@ def _get_ref(self, meeting=None, meeting_doc_refs=settings.MEETING_DOC_HREFS): # the earlier resolution order, but there's at the moment one single # instance which matches this (with correct results), so we won't # break things all over the place. - if not hasattr(self, '_cached_href'): + cache_attr = "_cached_href" if versioned else "_cached_versionless_href" + if not hasattr(self, cache_attr): validator = URLValidator() if self.external_url and self.external_url.split(':')[0] in validator.schemes: validator(self.external_url) return self.external_url + meeting_doc_refs = ( + settings.MEETING_DOC_HREFS if versioned else settings.MEETING_DOC_GREFS + ) if self.type_id in settings.DOC_HREFS and self.type_id in meeting_doc_refs: if self.meeting_related(): self.is_meeting_related = True @@ -301,8 +341,8 @@ def _get_ref(self, meeting=None, meeting_doc_refs=settings.MEETING_DOC_HREFS): if href.startswith('/'): href = settings.IDTRACKER_BASE_URL + href - self._cached_href = href - return self._cached_href + setattr(self, cache_attr, href) + return getattr(self, cache_attr) def set_state(self, state): """Switch state type implicit in state to state. This just @@ -407,9 +447,56 @@ def friendly_state(self): else: return state.name + def author_names(self): + """Author names as a list of strings""" + names = [] + if self.type_id == "rfc" and self.rfcauthor_set.exists(): + for author in self.rfcauthor_set.select_related("person"): + if author.person: + names.append(author.person.name) + else: + # titlepage_name cannot be blank + names.append(author.titlepage_name) + else: + names = [ + author.person.name + for author in self.documentauthor_set.select_related("person") + ] + return names + + def author_persons_or_names(self): + """Authors as a list of named tuples with person and/or titlepage_name""" + Author = namedtuple("Author", "person titlepage_name") + persons_or_names = [] + if self.type_id=="rfc" and self.rfcauthor_set.exists(): + for author in self.rfcauthor_set.select_related("person"): + persons_or_names.append(Author(person=author.person, titlepage_name=author.titlepage_name)) + else: + for author in self.documentauthor_set.select_related("person"): + persons_or_names.append(Author(person=author.person, titlepage_name="")) + return persons_or_names + + def author_persons(self): + """Authors as a list of Persons + + Omits any RfcAuthors with a null person field. + """ + if self.type_id == "rfc" and self.rfcauthor_set.exists(): + authors_qs = self.rfcauthor_set.filter(person__isnull=False) + else: + authors_qs = self.documentauthor_set.all() + return [a.person for a in authors_qs.select_related("person")] + def author_list(self): + """List of author emails""" + if self.type_id == "rfc" and self.rfcauthor_set.exists(): + author_qs = self.rfcauthor_set.select_related("person").order_by("order") + else: + author_qs = self.documentauthor_set.select_related("email").order_by( + "order" + ) best_addresses = [] - for author in self.documentauthor_set.all(): + for author in author_qs: if author.email: if author.email.active or not author.email.person: best_addresses.append(author.email.address) @@ -417,9 +504,6 @@ def author_list(self): best_addresses.append(author.email.person.email_address()) return ", ".join(best_addresses) - def authors(self): - return [ a.person for a in self.documentauthor_set.all() ] - # This, and several other ballot related functions here, assume that there is only one active ballot for a document at any point in time. # If that assumption is violated, they will only expose the most recently created ballot def ballot_open(self, ballot_type_slug): @@ -558,19 +642,7 @@ def text(self, size = -1): except IOError as e: log.log(f"Error reading text for {path}: {e}") return None - text = None - try: - text = raw.decode('utf-8') - except UnicodeDecodeError: - for back in range(1,4): - try: - text = raw[:-back].decode('utf-8') - break - except UnicodeDecodeError: - pass - if text is None: - text = raw.decode('latin-1') - return text + return decode_document_content(raw) def text_or_error(self): return self.text() or "Error; cannot read '%s'"%self.get_base_name() @@ -721,7 +793,14 @@ def referenced_by_rfcs_as_rfc_or_draft(self): if self.type_id == "rfc" and self.came_from_draft(): refs_to |= self.came_from_draft().referenced_by_rfcs() return refs_to - + + def sent_to_rfc_editor_event(self): + if self.stream_id == "ietf": + return self.docevent_set.filter(type="iesg_approved").order_by("-time").first() + elif self.stream_id in ["editorial", "iab", "irtf", "ise"]: + return self.docevent_set.filter(type="requested_publication").order_by("-time").first() + else: + return None class Meta: abstract = True @@ -845,6 +924,54 @@ def is_approved_downref(self): return False +class RfcAuthor(models.Model): + """Captures the authors of an RFC as represented on the RFC title page. + + This deviates from DocumentAuthor in that it does not get moved into the DocHistory + hierarchy as documents are saved. It will attempt to preserve email, country, and affiliation + from the DocumentAuthor objects associated with the draft leading to this RFC (which + may be wrong if the author moves or changes affiliation while the document is in the + queue). + + It does not, at this time, attempt to capture the authors from anything _but_ the title + page. The datatracker may know more about such authors based on information from the draft + leading to the RFC, and future work may take that into account. + + Once doc.rfcauthor_set.exists() for a doc of type `rfc`, doc.documentauthor_set should be + ignored. + """ + + document = ForeignKey( + "Document", + on_delete=models.CASCADE, + limit_choices_to={"type_id": "rfc"}, # only affects ModelForms (e.g., admin) + ) + titlepage_name = models.CharField(max_length=128, blank=False) + is_editor = models.BooleanField(default=False) + person = ForeignKey(Person, null=True, blank=True, on_delete=models.PROTECT) + affiliation = models.CharField(max_length=100, blank=True, help_text="Organization/company used by author for submission") + country = models.CharField(max_length=255, blank=True, help_text="Country used by author for submission") + order = models.IntegerField(default=1) + + def __str__(self): + return u"%s %s (%s)" % (self.document.name, self.person, self.order) + + class Meta: + ordering=["document", "order"] + indexes=[ + models.Index(fields=["document", "order"]) + ] + + @property + def email(self) -> Email | None: + return self.person.email() if self.person else None + + def format_for_titlepage(self): + if self.is_editor: + return f"{self.titlepage_name}, Ed." + return self.titlepage_name + + class DocumentAuthorInfo(models.Model): person = ForeignKey(Person) # email should only be null for some historic documents @@ -894,7 +1021,7 @@ class Meta: def role_for_doc(self): """Brief string description of this person's relationship to the doc""" roles = [] - if self.person in self.document.authors(): + if self.person in self.document.author_persons(): roles.append('Author') if self.person == self.document.ad: roles.append('Responsible AD') @@ -913,13 +1040,25 @@ def role_for_doc(self): roles.append('Action Holder') return ', '.join(roles) +# N.B., at least a couple dozen documents exist that do not satisfy this validator validate_docname = RegexValidator( r'^[-a-z0-9]+$', "Provide a valid document name consisting of lowercase letters, numbers and hyphens.", 'invalid' ) + +SUBSERIES_DOC_TYPE_IDS = ("bcp", "fyi", "std") + + +class DocumentQuerySet(models.QuerySet): + def subseries_docs(self): + return self.filter(type_id__in=SUBSERIES_DOC_TYPE_IDS) + + class Document(StorableMixin, DocumentInfo): + objects = DocumentQuerySet.as_manager() + name = models.CharField(max_length=255, validators=[validate_docname,], unique=True) # immutable action_holders = models.ManyToManyField(Person, through=DocumentActionHolder, blank=True) @@ -1025,6 +1164,22 @@ def request_closed_time(self, review_req): e = self.latest_event(ReviewRequestDocEvent, type="closed_review_request", review_request=review_req) return e.time if e and e.time else None + @property + def area(self) -> Group | None: + """Get area for document, if one exists + + None for non-IETF-stream documents. N.b., this is stricter than Group.area() and + uses different logic from Document.area_acronym(). + """ + if self.stream_id != "ietf": + return None + if self.group is None: + return None + parent = self.group.parent + if parent.type_id == "area": + return parent + return None + def area_acronym(self): g = self.group if g: @@ -1120,11 +1275,8 @@ def submission(self): s = s.first() return s - def pub_date(self): - """Get the publication date for this document - - This is the rfc publication date for RFCs, and the new-revision date for other documents. - """ + def pub_datetime(self): + """Get the publication datetime of this document""" if self.type_id == "rfc": # As of Sept 2022, in ietf.sync.rfceditor.update_docs_from_rfc_index() `published_rfc` events are # created with a timestamp whose date *in the PST8PDT timezone* is the official publication date @@ -1132,7 +1284,15 @@ def pub_date(self): event = self.latest_event(type='published_rfc') else: event = self.latest_event(type='new_revision') - return event.time.astimezone(RPC_TZINFO).date() if event else None + return event.time.astimezone(RPC_TZINFO) if event else None + + def pub_date(self): + """Get the publication date for this document + + This is the rfc publication date for RFCs, and the new-revision date for other documents. + """ + pub_datetime = self.pub_datetime() + return None if pub_datetime is None else pub_datetime.date() def is_dochistory(self): return False @@ -1157,7 +1317,7 @@ def fake_history_obj(self, rev): elif rev_events.exists(): time = rev_events.first().time else: - time = datetime.datetime.fromtimestamp(0, datetime.timezone.utc) + time = datetime.datetime.fromtimestamp(0, datetime.UTC) dh = DocHistory(name=self.name, rev=rev, doc=self, time=time, type=self.type, title=self.title, stream=self.stream, group=self.group) @@ -1168,6 +1328,32 @@ def action_holders_enabled(self): iesg_state = self.get_state('draft-iesg') return iesg_state and iesg_state.slug != 'idexists' + def formats(self): + """List of file formats available + + Only implemented for RFCs. Relies on StoredObject. + """ + if self.type_id != "rfc": + raise RuntimeError("Only allowed for type=rfc") + + # StoredObject.doc_rev can be null or "" to represent no rev. Match either + # of these when self.rev is "" (always expected to be the case for RFCs) + rev_q = Q(doc_rev=self.rev) + if self.rev == "": + rev_q |= Q(doc_rev__isnull=True) + return [ + { + "fmt": Path(object_name).parts[0], + "name": object_name, + } + for object_name in StoredObject.objects.filter( + rev_q, + store="rfc", + doc_name=self.name, + ).values_list("name", flat=True) + ] + + class DocumentURL(models.Model): doc = ForeignKey(Document) tag = ForeignKey(DocUrlTagName) @@ -1360,6 +1546,7 @@ class DocReminder(models.Model): ("rfc_editor_received_announcement", "Announcement was received by RFC Editor"), ("requested_publication", "Publication at RFC Editor requested"), ("sync_from_rfc_editor", "Received updated information from RFC Editor"), + ("changed_rpc_assignments", "Changed RPC queue assignments"), # review ("requested_review", "Requested review"), @@ -1425,6 +1612,9 @@ class StateDocEvent(DocEvent): class ConsensusDocEvent(DocEvent): consensus = models.BooleanField(null=True, default=None) +class RpcAssignmentDocEvent(DocEvent): + assignments = models.TextField(blank=True) + # IESG events class BallotType(models.Model): doc_type = ForeignKey(DocTypeName, blank=True, null=True) @@ -1580,6 +1770,11 @@ class EditedAuthorsDocEvent(DocEvent): """ basis = models.CharField(help_text="What is the source or reasoning for the changes to the author list",max_length=255) + +class EditedRfcAuthorsDocEvent(DocEvent): + """Change to the RfcAuthor list for a document""" + + class BofreqEditorDocEvent(DocEvent): """ Capture the proponents of a BOF Request.""" editors = models.ManyToManyField('person.Person', blank=True) @@ -1588,9 +1783,17 @@ class BofreqResponsibleDocEvent(DocEvent): """ Capture the responsible leadership (IAB and IESG members) for a BOF Request """ responsible = models.ManyToManyField('person.Person', blank=True) + +class StoredObjectQuerySet(models.QuerySet): + def exclude_deleted(self): + return self.filter(deleted__isnull=True) + + class StoredObject(models.Model): """Hold metadata about objects placed in object storage""" + objects = StoredObjectQuerySet.as_manager() + store = models.CharField(max_length=256) name = models.CharField(max_length=1024, null=False, blank=False) # N.B. the 1024 limit on name comes from S3 sha384 = models.CharField(max_length=96) @@ -1615,3 +1818,6 @@ class Meta: indexes = [ models.Index(fields=["doc_name", "doc_rev"]), ] + + def __str__(self): + return f"{self.store}:{self.name}" diff --git a/ietf/doc/resources.py b/ietf/doc/resources.py index 157a3ad556..9da7cb57d8 100644 --- a/ietf/doc/resources.py +++ b/ietf/doc/resources.py @@ -17,8 +17,9 @@ InitialReviewDocEvent, DocHistoryAuthor, BallotDocEvent, RelatedDocument, RelatedDocHistory, BallotPositionDocEvent, AddedMessageEvent, SubmissionDocEvent, ReviewRequestDocEvent, ReviewAssignmentDocEvent, EditedAuthorsDocEvent, DocumentURL, - IanaExpertDocEvent, IRSGBallotDocEvent, DocExtResource, DocumentActionHolder, - BofreqEditorDocEvent, BofreqResponsibleDocEvent, StoredObject) + IanaExpertDocEvent, IRSGBallotDocEvent, DocExtResource, DocumentActionHolder, + BofreqEditorDocEvent, BofreqResponsibleDocEvent, StoredObject, RfcAuthor, + EditedRfcAuthorsDocEvent, RpcAssignmentDocEvent) from ietf.name.resources import BallotPositionNameResource, DocTypeNameResource class BallotTypeResource(ModelResource): @@ -650,6 +651,31 @@ class Meta: api.doc.register(EditedAuthorsDocEventResource()) + +from ietf.person.resources import PersonResource +class EditedRfcAuthorsDocEventResource(ModelResource): + by = ToOneField(PersonResource, 'by') + doc = ToOneField(DocumentResource, 'doc') + docevent_ptr = ToOneField(DocEventResource, 'docevent_ptr') + class Meta: + queryset = EditedRfcAuthorsDocEvent.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'editedrfcauthorsdocevent' + ordering = ['id', ] + filtering = { + "id": ALL, + "time": ALL, + "type": ALL, + "rev": ALL, + "desc": ALL, + "by": ALL_WITH_RELATIONS, + "doc": ALL_WITH_RELATIONS, + "docevent_ptr": ALL_WITH_RELATIONS, + } +api.doc.register(EditedRfcAuthorsDocEventResource()) + + from ietf.name.resources import DocUrlTagNameResource class DocumentURLResource(ModelResource): doc = ToOneField(DocumentResource, 'doc') @@ -865,3 +891,53 @@ class Meta: "deleted": ALL, } api.doc.register(StoredObjectResource()) + + +from ietf.person.resources import EmailResource, PersonResource +class RfcAuthorResource(ModelResource): + document = ToOneField(DocumentResource, 'document') + person = ToOneField(PersonResource, 'person', null=True) + email = ToOneField(EmailResource, 'email', null=True, readonly=True) + class Meta: + queryset = RfcAuthor.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'rfcauthor' + ordering = ['id', ] + filtering = { + "id": ALL, + "titlepage_name": ALL, + "is_editor": ALL, + "affiliation": ALL, + "country": ALL, + "order": ALL, + "document": ALL_WITH_RELATIONS, + "person": ALL_WITH_RELATIONS, + "email": ALL_WITH_RELATIONS, + } +api.doc.register(RfcAuthorResource()) + + +from ietf.person.resources import PersonResource +class RpcAssignmentDocEventResource(ModelResource): + by = ToOneField(PersonResource, 'by') + doc = ToOneField(DocumentResource, 'doc') + docevent_ptr = ToOneField(DocEventResource, 'docevent_ptr') + class Meta: + queryset = RpcAssignmentDocEvent.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'rpcassignmentdocevent' + ordering = ['docevent_ptr', ] + filtering = { + "id": ALL, + "time": ALL, + "type": ALL, + "rev": ALL, + "desc": ALL, + "assignments": ALL, + "by": ALL_WITH_RELATIONS, + "doc": ALL_WITH_RELATIONS, + "docevent_ptr": ALL_WITH_RELATIONS, + } +api.doc.register(RpcAssignmentDocEventResource()) diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py new file mode 100644 index 0000000000..3651670962 --- /dev/null +++ b/ietf/doc/serializers.py @@ -0,0 +1,360 @@ +# Copyright The IETF Trust 2024-2026, All Rights Reserved +"""django-rest-framework serializers""" + +from dataclasses import dataclass +from typing import Literal, ClassVar + +from django.db.models.manager import BaseManager +from django.db.models.query import QuerySet +from drf_spectacular.utils import extend_schema_field +from rest_framework import serializers + +from ietf.group.serializers import ( + AreaDirectorSerializer, + AreaSerializer, + GroupSerializer, +) +from ietf.name.serializers import StreamNameSerializer +from ietf.utils import log +from .models import Document, DocumentAuthor, RfcAuthor + + +class RfcAuthorSerializer(serializers.ModelSerializer): + """Serializer for an RfcAuthor / DocumentAuthor in a response""" + + email = serializers.EmailField(source="email.address", read_only=True) + datatracker_person_path = serializers.URLField( + source="person.get_absolute_url", + required=False, + help_text="URL for person link (relative to datatracker base URL)", + read_only=True, + ) + + class Meta: + model = RfcAuthor + fields = [ + "titlepage_name", + "is_editor", + "person", + "email", + "affiliation", + "country", + "datatracker_person_path", + ] + + def to_representation(self, instance): + """instance -> primitive data types + + Translates a DocumentAuthor into an equivalent RfcAuthor we can use the same + serializer for either type. + """ + if isinstance(instance, DocumentAuthor): + # create a non-persisted RfcAuthor as a shim - do not save it! + document_author = instance + instance = RfcAuthor( + titlepage_name=document_author.person.plain_name(), + is_editor=False, + person=document_author.person, + affiliation=document_author.affiliation, + country=document_author.country, + order=document_author.order, + ) + return super().to_representation(instance) + + def validate(self, data): + email = data.get("email") + if email is not None: + person = data.get("person") + if person is None: + raise serializers.ValidationError( + { + "email": "cannot have an email without a person", + }, + code="email-without-person", + ) + if email.person_id != person.pk: + raise serializers.ValidationError( + { + "email": "email must belong to person", + }, + code="email-person-mismatch", + ) + return data + + +@dataclass +class DocIdentifier: + type: Literal["doi", "issn"] + value: str + + +class DocIdentifierSerializer(serializers.Serializer): + type = serializers.ChoiceField(choices=["doi", "issn"]) + value = serializers.CharField() + + +type RfcStatusSlugT = Literal[ + "std", + "ps", + "ds", + "bcp", + "inf", + "exp", + "hist", + "unkn", + "not-issued", +] + + +@dataclass +class RfcStatus: + """Helper to extract the 'Status' from an RFC document for serialization""" + + slug: RfcStatusSlugT + + # Names that aren't just the slug itself. ClassVar annotation prevents dataclass from treating this as a field. + fancy_names: ClassVar[dict[RfcStatusSlugT, str]] = { + "std": "internet standard", + "ps": "proposed standard", + "ds": "draft standard", + "bcp": "best current practice", + "inf": "informational", + "exp": "experimental", + "hist": "historic", + "unkn": "unknown", + } + + # ClassVar annotation prevents dataclass from treating this as a field + stdlevelname_slug_map: ClassVar[dict[str, RfcStatusSlugT]] = { + "bcp": "bcp", + "ds": "ds", + "exp": "exp", + "hist": "hist", + "inf": "inf", + "std": "std", + "ps": "ps", + "unkn": "unkn", + } + + # ClassVar annotation prevents dataclass from treating this as a field + status_slugs: ClassVar[list[RfcStatusSlugT]] = sorted( + # TODO implement "not-issued" RFCs + set(stdlevelname_slug_map.values()) | {"not-issued"} + ) + + @property + def name(self): + return RfcStatus.fancy_names.get(self.slug, self.slug) + + @classmethod + def from_document(cls, doc: Document): + """Decide the status that applies to a document""" + return cls( + slug=(cls.stdlevelname_slug_map.get(doc.std_level.slug, "unkn")), + ) + + @classmethod + def filter(cls, queryset, name, value: list[RfcStatusSlugT]): + """Filter a queryset by status + + This is basically the inverse of the from_document() method. Given a status name, filter + the queryset to those in that status. The queryset should be a Document queryset. + """ + interesting_slugs = [ + stdlevelname_slug + for stdlevelname_slug, status_slug in cls.stdlevelname_slug_map.items() + if status_slug in value + ] + if len(interesting_slugs) == 0: + return queryset.none() + return queryset.filter(std_level__slug__in=interesting_slugs) + + +class RfcStatusSerializer(serializers.Serializer): + """Status serializer for a Document instance""" + + slug = serializers.ChoiceField(choices=RfcStatus.status_slugs) + name = serializers.CharField() + + def to_representation(self, instance: Document): + return super().to_representation(instance=RfcStatus.from_document(instance)) + + +class ShepherdSerializer(serializers.Serializer): + email = serializers.EmailField(source="email_address") + + +class RelatedDraftSerializer(serializers.Serializer): + id = serializers.IntegerField(source="source.id") + name = serializers.CharField(source="source.name") + title = serializers.CharField(source="source.title") + shepherd = ShepherdSerializer(source="source.shepherd", allow_null=True) + ad = AreaDirectorSerializer(source="source.ad", allow_null=True) + + +class RelatedRfcSerializer(serializers.Serializer): + id = serializers.IntegerField(source="target.id") + number = serializers.IntegerField(source="target.rfc_number") + title = serializers.CharField(source="target.title") + + +class ReverseRelatedRfcSerializer(serializers.Serializer): + id = serializers.IntegerField(source="source.id") + number = serializers.IntegerField(source="source.rfc_number") + title = serializers.CharField(source="source.title") + + +class ContainingSubseriesSerializer(serializers.Serializer): + name = serializers.CharField(source="source.name") + type = serializers.CharField(source="source.type_id") + + +class RfcFormatSerializer(serializers.Serializer): + RFC_FORMATS = ("xml", "txt", "html", "pdf", "ps", "json", "notprepped") + + fmt = serializers.ChoiceField(choices=RFC_FORMATS) + name = serializers.CharField(help_text="Name of blob in the blob store") + + +class RfcMetadataSerializer(serializers.ModelSerializer): + """Serialize metadata of an RFC + + This needs to be called with a Document queryset that has been processed with + api.augment_rfc_queryset() or it very likely will not work. Some of the typing + refers to Document, but this should really be WithAnnotations[Document, ...]. + However, have not been able to make that work yet. + """ + + number = serializers.IntegerField(source="rfc_number") + published = serializers.DateField() + status = RfcStatusSerializer(source="*") + authors = serializers.SerializerMethodField() + group = GroupSerializer() + area = AreaSerializer(read_only=True) + stream = StreamNameSerializer() + ad = AreaDirectorSerializer(read_only=True, allow_null=True) + group_list_email = serializers.EmailField(source="group.list_email", read_only=True) + identifiers = serializers.SerializerMethodField() + draft = serializers.SerializerMethodField() + obsoletes = RelatedRfcSerializer(many=True, read_only=True) + obsoleted_by = ReverseRelatedRfcSerializer(many=True, read_only=True) + updates = RelatedRfcSerializer(many=True, read_only=True) + updated_by = ReverseRelatedRfcSerializer(many=True, read_only=True) + subseries = ContainingSubseriesSerializer(many=True, read_only=True) + formats = RfcFormatSerializer( + many=True, read_only=True, help_text="Available formats" + ) + keywords = serializers.ListField(child=serializers.CharField(), read_only=True) + has_errata = serializers.BooleanField(read_only=True) + + class Meta: + model = Document + fields = [ + "number", + "title", + "published", + "status", + "pages", + "authors", + "group", + "area", + "stream", + "ad", + "group_list_email", + "identifiers", + "obsoletes", + "obsoleted_by", + "updates", + "updated_by", + "subseries", + "draft", + "abstract", + "formats", + "keywords", + "has_errata", + ] + + @extend_schema_field(RfcAuthorSerializer(many=True)) + def get_authors(self, doc: Document): + # If doc has any RfcAuthors, use those, otherwise fall back to DocumentAuthors + author_queryset: QuerySet[RfcAuthor] | QuerySet[DocumentAuthor] = ( + doc.rfcauthor_set.all() + if doc.rfcauthor_set.exists() + else doc.documentauthor_set.all() + ) + # RfcAuthorSerializer can deal with DocumentAuthor instances + return RfcAuthorSerializer( + instance=author_queryset, + many=True, + ).data + + @extend_schema_field(DocIdentifierSerializer(many=True)) + def get_identifiers(self, doc: Document): + identifiers = [] + if doc.doi: + identifiers.append( + DocIdentifier(type="doi", value=doc.doi) + ) + return DocIdentifierSerializer(instance=identifiers, many=True).data + + @extend_schema_field(RelatedDraftSerializer) + def get_draft(self, doc: Document): + if hasattr(doc, "drafts"): + # This is the expected case - drafts is added by a Prefetch in + # the augment_rfc_queryset() method. + try: + related_doc = doc.drafts[0] + except IndexError: + return None + else: + # Fallback in case augment_rfc_queryset() was not called + log.log( + f"Warning: {self.__class__}.get_draft() called without prefetched draft" + ) + related_doc = doc.came_from_draft() + return RelatedDraftSerializer(related_doc).data + + +class RfcSerializer(RfcMetadataSerializer): + """Serialize an RFC, including its metadata and text content if available""" + + text = serializers.CharField(allow_null=True) + + class Meta: + model = RfcMetadataSerializer.Meta.model + fields = RfcMetadataSerializer.Meta.fields + ["text"] + + +class SubseriesContentListSerializer(serializers.ListSerializer): + """ListSerializer that gets its object from item.target""" + + def to_representation(self, data): + """ + List of object instances -> List of dicts of primitive datatypes. + """ + # Dealing with nested relationships, data can be a Manager, + # so, first get a queryset from the Manager if needed + iterable = data.all() if isinstance(data, BaseManager) else data + # Serialize item.target instead of item itself + return [self.child.to_representation(item.target) for item in iterable] + + +class SubseriesContentSerializer(RfcMetadataSerializer): + """Serialize RFC contained in a subseries doc""" + + class Meta(RfcMetadataSerializer.Meta): + list_serializer_class = SubseriesContentListSerializer + + +class SubseriesDocSerializer(serializers.ModelSerializer): + """Serialize a subseries document (e.g., a BCP or STD)""" + + contents = SubseriesContentSerializer(many=True) + + class Meta: + model = Document + fields = [ + "name", + "type", + "contents", + ] diff --git a/ietf/doc/storage.py b/ietf/doc/storage.py new file mode 100644 index 0000000000..ee1e76c4fa --- /dev/null +++ b/ietf/doc/storage.py @@ -0,0 +1,181 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +from typing import Optional + +import debug # pyflakes:ignore +import json + +from contextlib import contextmanager +from storages.backends.s3 import S3Storage + +from django.core.files.base import File + +from ietf.blobdb.storage import BlobdbStorage +from ietf.doc.models import StoredObject +from ietf.utils.log import log +from ietf.utils.storage import MetadataFile +from ietf.utils.timezone import timezone + + +class StoredObjectFile(MetadataFile): + """Django storage File object that represents a StoredObject""" + def __init__(self, file, name, mtime=None, content_type="", store=None, doc_name=None, doc_rev=None): + super().__init__( + file=file, + name=name, + mtime=mtime, + content_type=content_type, + ) + self.store = store + self.doc_name = doc_name + self.doc_rev = doc_rev + + @classmethod + def from_storedobject(cls, file, name, store): + """Alternate constructor for objects that already exist in the StoredObject table""" + stored_object = StoredObject.objects.exclude_deleted().filter(store=store, name=name).first() + if stored_object is None: + raise FileNotFoundError(f"StoredObject for {store}:{name} does not exist or was deleted") + file = cls(file, name, store, doc_name=stored_object.doc_name, doc_rev=stored_object.doc_rev) + if int(file.custom_metadata["len"]) != stored_object.len: + raise RuntimeError(f"File length changed unexpectedly for {store}:{name}") + if file.custom_metadata["sha384"] != stored_object.sha384: + raise RuntimeError(f"SHA-384 hash changed unexpectedly for {store}:{name}") + return file + + +@contextmanager +def maybe_log_timing(enabled, op, **kwargs): + """If enabled, log elapsed time and additional data from kwargs + + Emits log even if an exception occurs + """ + before = timezone.now() + exception = None + try: + yield + except Exception as err: + exception = err + raise + finally: + if enabled: + dt = timezone.now() - before + log( + json.dumps( + { + "log": "S3Storage_timing", + "seconds": dt.total_seconds(), + "op": op, + "exception": "" if exception is None else repr(exception), + **kwargs, + } + ) + ) + + +class MetadataS3Storage(S3Storage): + def get_default_settings(self): + # add a default for the ietf_log_blob_timing boolean + return super().get_default_settings() | {"ietf_log_blob_timing": False} + + def _save(self, name, content: File): + with maybe_log_timing( + self.ietf_log_blob_timing, "_save", bucket_name=self.bucket_name, name=name + ): + return super()._save(name, content) + + def _open(self, name, mode="rb"): + with maybe_log_timing( + self.ietf_log_blob_timing, + "_open", + bucket_name=self.bucket_name, + name=name, + mode=mode, + ): + return super()._open(name, mode) + + def delete(self, name): + with maybe_log_timing( + self.ietf_log_blob_timing, "delete", bucket_name=self.bucket_name, name=name + ): + super().delete(name) + + def _get_write_parameters(self, name, content=None): + # debug.show('f"getting write parameters for {name}"') + params = super()._get_write_parameters(name, content) + # If we have a non-empty explicit content type, use it + content_type = getattr(content, "content_type", "").strip() + if content_type != "": + params["ContentType"] = content_type + if "Metadata" not in params: + params["Metadata"] = {} + if hasattr(content, "custom_metadata"): + params["Metadata"].update(content.custom_metadata) + return params + + +class StoredObjectBlobdbStorage(BlobdbStorage): + warn_if_missing = True # TODO-BLOBSTORE make this configurable (or remove it) + + def _save_stored_object(self, name, content) -> StoredObject: + now = timezone.now() + record, created = StoredObject.objects.get_or_create( + store=self.bucket_name, + name=name, + defaults=dict( + sha384=content.custom_metadata["sha384"], + len=int(content.custom_metadata["len"]), + store_created=now, + created=now, + modified=now, + doc_name=getattr( + content, + "doc_name", # Note that these are assumed to be invariant + None, # should be blank? + ), + doc_rev=getattr( + content, + "doc_rev", # for a given name + None, # should be blank? + ), + ), + ) + if not created and ( + record.sha384 != content.custom_metadata["sha384"] + or record.len != int(content.custom_metadata["len"]) + or record.deleted is not None + ): + record.sha384 = content.custom_metadata["sha384"] + record.len = int(content.custom_metadata["len"]) + record.modified = now + record.deleted = None + record.save() + return record + + def _delete_stored_object(self, name) -> Optional[StoredObject]: + existing_record = StoredObject.objects.filter(store=self.bucket_name, name=name) + if not existing_record.exists() and self.warn_if_missing: + complaint = ( + f"WARNING: Asked to delete {name} from {self.bucket_name} storage, " + f"but there was no matching StoredObject" + ) + log(complaint) + debug.show("complaint") + else: + now = timezone.now() + # Note that existing_record is a queryset that will have one matching object + existing_record.exclude_deleted().update(deleted=now) + return existing_record.first() + + def _save(self, name, content): + """Perform the save operation + + In principle the name could change on save to the blob store. As of now, BlobdbStorage + will not change it, but allow for that possibility. Callers should be prepared for this. + """ + saved_name = super()._save(name, content) + self._save_stored_object(saved_name, content) + return saved_name + + def delete(self, name): + self._delete_stored_object(name) + super().delete(name) diff --git a/ietf/doc/storage_backends.py b/ietf/doc/storage_backends.py deleted file mode 100644 index 5eeab040e5..0000000000 --- a/ietf/doc/storage_backends.py +++ /dev/null @@ -1,192 +0,0 @@ -# Copyright The IETF Trust 2025, All Rights Reserved - -import debug # pyflakes:ignore -import json - -from contextlib import contextmanager -from hashlib import sha384 -from io import BufferedReader -from storages.backends.s3 import S3Storage -from typing import Optional, Union - -from django.core.files.base import File - -from ietf.doc.models import StoredObject -from ietf.utils.log import log -from ietf.utils.timezone import timezone - - -@contextmanager -def maybe_log_timing(enabled, op, **kwargs): - """If enabled, log elapsed time and additional data from kwargs - - Emits log even if an exception occurs - """ - before = timezone.now() - exception = None - try: - yield - except Exception as err: - exception = err - raise - finally: - if enabled: - dt = timezone.now() - before - log( - json.dumps( - { - "log": "S3Storage_timing", - "seconds": dt.total_seconds(), - "op": op, - "exception": "" if exception is None else repr(exception), - **kwargs, - } - ) - ) - - -# TODO-BLOBSTORE -# Consider overriding save directly so that -# we capture metadata for, e.g., ImageField objects -class CustomS3Storage(S3Storage): - - def __init__(self, **settings): - self.in_flight_custom_metadata = {} # type is Dict[str, Dict[str, str]] - super().__init__(**settings) - - def get_default_settings(self): - # add a default for the ietf_log_blob_timing boolean - return super().get_default_settings() | {"ietf_log_blob_timing": False} - - def _save(self, name, content): - with maybe_log_timing( - self.ietf_log_blob_timing, "_save", bucket_name=self.bucket_name, name=name - ): - return super()._save(name, content) - - def _open(self, name, mode="rb"): - with maybe_log_timing( - self.ietf_log_blob_timing, - "_open", - bucket_name=self.bucket_name, - name=name, - mode=mode, - ): - return super()._open(name, mode) - - def delete(self, name): - with maybe_log_timing( - self.ietf_log_blob_timing, "delete", bucket_name=self.bucket_name, name=name - ): - super().delete(name) - - def store_file( - self, - kind: str, - name: str, - file: Union[File, BufferedReader], - allow_overwrite: bool = False, - doc_name: Optional[str] = None, - doc_rev: Optional[str] = None, - ): - is_new = not self.exists_in_storage(kind, name) - # debug.show('f"Asked to store {name} in {kind}: is_new={is_new}, allow_overwrite={allow_overwrite}"') - if not allow_overwrite and not is_new: - log(f"Failed to save {kind}:{name} - name already exists in store") - debug.show('f"Failed to save {kind}:{name} - name already exists in store"') - # raise Exception("Not ignoring overwrite attempts while testing") - else: - try: - new_name = self.save(name, file) - now = timezone.now() - record, created = StoredObject.objects.get_or_create( - store=kind, - name=name, - defaults=dict( - sha384=self.in_flight_custom_metadata[name]["sha384"], - len=int(self.in_flight_custom_metadata[name]["len"]), - store_created=now, - created=now, - modified=now, - doc_name=doc_name, # Note that these are assumed to be invariant - doc_rev=doc_rev, # for a given name - ), - ) - if not created: - record.sha384 = self.in_flight_custom_metadata[name]["sha384"] - record.len = int(self.in_flight_custom_metadata[name]["len"]) - record.modified = now - record.deleted = None - record.save() - if new_name != name: - complaint = f"Error encountered saving '{name}' - results stored in '{new_name}' instead." - log(complaint) - debug.show("complaint") - # Note that we are otherwise ignoring this condition - it should become an error later. - except Exception as e: - # Log and then swallow the exception while we're learning. - # Don't let failure pass so quietly when these are the autoritative bits. - complaint = f"Failed to save {kind}:{name}" - log(complaint, e) - debug.show('f"{complaint}: {e}"') - finally: - del self.in_flight_custom_metadata[name] - return None - - def exists_in_storage(self, kind: str, name: str) -> bool: - try: - # open is realized with a HEAD - # See https://github.com/jschneier/django-storages/blob/b79ea310201e7afd659fe47e2882fe59aae5b517/storages/backends/s3.py#L528 - with self.open(name): - return True - except FileNotFoundError: - return False - - def remove_from_storage( - self, kind: str, name: str, warn_if_missing: bool = True - ) -> None: - now = timezone.now() - try: - with self.open(name): - pass - self.delete(name) - # debug.show('f"deleted {name} from {kind} storage"') - except FileNotFoundError: - if warn_if_missing: - complaint = ( - f"WARNING: Asked to delete non-existent {name} from {kind} storage" - ) - log(complaint) - debug.show("complaint") - existing_record = StoredObject.objects.filter(store=kind, name=name) - if not existing_record.exists() and warn_if_missing: - complaint = f"WARNING: Asked to delete {name} from {kind} storage, but there was no matching StorageObject" - log(complaint) - debug.show("complaint") - else: - # Note that existing_record is a queryset that will have one matching object - existing_record.filter(deleted__isnull=True).update(deleted=now) - - def _get_write_parameters(self, name, content=None): - # debug.show('f"getting write parameters for {name}"') - params = super()._get_write_parameters(name, content) - if "Metadata" not in params: - params["Metadata"] = {} - try: - content.seek(0) - except AttributeError: # TODO-BLOBSTORE - debug.say("Encountered Non-Seekable content") - raise NotImplementedError("cannot handle unseekable content") - content_bytes = content.read() - if not isinstance( - content_bytes, bytes - ): # TODO-BLOBSTORE: This is sketch-development only -remove before committing - raise Exception(f"Expected bytes - got {type(content_bytes)}") - content.seek(0) - metadata = { - "len": f"{len(content_bytes)}", - "sha384": f"{sha384(content_bytes).hexdigest()}", - } - params["Metadata"].update(metadata) - self.in_flight_custom_metadata[name] = metadata - return params diff --git a/ietf/doc/storage_utils.py b/ietf/doc/storage_utils.py index 012efc9071..c7cc6989cd 100644 --- a/ietf/doc/storage_utils.py +++ b/ietf/doc/storage_utils.py @@ -1,21 +1,28 @@ # Copyright The IETF Trust 2025, All Rights Reserved - +import datetime from io import BufferedReader from typing import Optional, Union + import debug # pyflakes ignore from django.conf import settings from django.core.files.base import ContentFile, File -from django.core.files.storage import storages +from django.core.files.storage import storages, Storage from ietf.utils.log import log +from ietf.utils.text import decode_document_content + + +class StorageUtilsError(Exception): + pass + +class AlreadyExistsError(StorageUtilsError): + pass -# TODO-BLOBSTORE (Future, maybe after leaving 3.9) : add a return type -def _get_storage(kind: str): - if kind in settings.MORE_STORAGE_NAMES: - # TODO-BLOBSTORE - add a checker that verifies configuration will only return CustomS3Storages +def _get_storage(kind: str) -> Storage: + if kind in settings.ARTIFACT_STORAGE_NAMES: return storages[kind] else: debug.say(f"Got into not-implemented looking for {kind}") @@ -26,23 +33,35 @@ def exists_in_storage(kind: str, name: str) -> bool: if settings.ENABLE_BLOBSTORAGE: try: store = _get_storage(kind) - return store.exists_in_storage(kind, name) + with store.open(name): + return True + except FileNotFoundError: + return False except Exception as err: log(f"Blobstore Error: Failed to test existence of {kind}:{name}: {repr(err)}") + if settings.SERVER_MODE == "development": + raise return False def remove_from_storage(kind: str, name: str, warn_if_missing: bool = True) -> None: if settings.ENABLE_BLOBSTORAGE: try: - store = _get_storage(kind) - store.remove_from_storage(kind, name, warn_if_missing) + if exists_in_storage(kind, name): + _get_storage(kind).delete(name) + elif warn_if_missing: + complaint = ( + f"WARNING: Asked to delete non-existent {name} from {kind} storage" + ) + debug.show("complaint") + log(complaint) except Exception as err: log(f"Blobstore Error: Failed to remove {kind}:{name}: {repr(err)}") + if settings.SERVER_MODE == "development": + raise return None -# TODO-BLOBSTORE: Try to refactor `kind` out of the signature of the methods already on the custom store (which knows its kind) def store_file( kind: str, name: str, @@ -50,14 +69,36 @@ def store_file( allow_overwrite: bool = False, doc_name: Optional[str] = None, doc_rev: Optional[str] = None, + content_type: str="", + mtime: Optional[datetime.datetime]=None, ) -> None: - # debug.show('f"asked to store {name} into {kind}"') + from .storage import StoredObjectFile # avoid circular import if settings.ENABLE_BLOBSTORAGE: try: - store = _get_storage(kind) - store.store_file(kind, name, file, allow_overwrite, doc_name, doc_rev) + is_new = not exists_in_storage(kind, name) + # debug.show('f"Asked to store {name} in {kind}: is_new={is_new}, allow_overwrite={allow_overwrite}"') + if not allow_overwrite and not is_new: + debug.show('f"Failed to save {kind}:{name} - name already exists in store"') + raise AlreadyExistsError(f"Failed to save {kind}:{name} - name already exists in store") + new_name = _get_storage(kind).save( + name, + StoredObjectFile( + file=file, + name=name, + doc_name=doc_name, + doc_rev=doc_rev, + mtime=mtime, + content_type=content_type, + ), + ) + if new_name != name: + complaint = f"Error encountered saving '{name}' - results stored in '{new_name}' instead." + debug.show("complaint") + raise StorageUtilsError(complaint) except Exception as err: log(f"Blobstore Error: Failed to store file {kind}:{name}: {repr(err)}") + if settings.SERVER_MODE == "development": + raise # TODO-BLOBSTORE eventually make this an error for all modes return None @@ -68,13 +109,26 @@ def store_bytes( allow_overwrite: bool = False, doc_name: Optional[str] = None, doc_rev: Optional[str] = None, + content_type: str = "", + mtime: Optional[datetime.datetime] = None, ) -> None: if settings.ENABLE_BLOBSTORAGE: try: - store_file(kind, name, ContentFile(content), allow_overwrite) + store_file( + kind, + name, + ContentFile(content), + allow_overwrite, + doc_name, + doc_rev, + content_type, + mtime, + ) except Exception as err: # n.b., not likely to get an exception here because store_file or store_bytes will catch it log(f"Blobstore Error: Failed to store bytes to {kind}:{name}: {repr(err)}") + if settings.SERVER_MODE == "development": + raise # TODO-BLOBSTORE eventually make this an error for all modes return None @@ -85,43 +139,65 @@ def store_str( allow_overwrite: bool = False, doc_name: Optional[str] = None, doc_rev: Optional[str] = None, + content_type: str = "", + mtime: Optional[datetime.datetime] = None, ) -> None: if settings.ENABLE_BLOBSTORAGE: try: content_bytes = content.encode("utf-8") - store_bytes(kind, name, content_bytes, allow_overwrite) + store_bytes( + kind, + name, + content_bytes, + allow_overwrite, + doc_name, + doc_rev, + content_type, + mtime, + ) except Exception as err: # n.b., not likely to get an exception here because store_file or store_bytes will catch it log(f"Blobstore Error: Failed to store string to {kind}:{name}: {repr(err)}") + if settings.SERVER_MODE == "development": + raise # TODO-BLOBSTORE eventually make this an error for all modes return None def retrieve_bytes(kind: str, name: str) -> bytes: - from ietf.doc.storage_backends import maybe_log_timing - content = b"" - if settings.ENABLE_BLOBSTORAGE: - try: - store = _get_storage(kind) - with store.open(name) as f: - with maybe_log_timing( - hasattr(store, "ietf_log_blob_timing") and store.ietf_log_blob_timing, - "read", - bucket_name=store.bucket_name if hasattr(store, "bucket_name") else "", - name=name, - ): - content = f.read() - except Exception as err: - log(f"Blobstore Error: Failed to read bytes from {kind}:{name}: {repr(err)}") + from ietf.doc.storage import maybe_log_timing + if not settings.ENABLE_BLOBSTORAGE: + return b"" + try: + store = _get_storage(kind) + with store.open(name) as f: + with maybe_log_timing( + hasattr(store, "ietf_log_blob_timing") and store.ietf_log_blob_timing, + "read", + bucket_name=store.bucket_name if hasattr(store, "bucket_name") else "", + name=name, + ): + content = f.read() + except Exception as err: + log(f"Blobstore Error: Failed to read bytes from {kind}:{name}: {repr(err)}") + raise return content def retrieve_str(kind: str, name: str) -> str: - content = "" - if settings.ENABLE_BLOBSTORAGE: - try: - content_bytes = retrieve_bytes(kind, name) - # TODO-BLOBSTORE: try to decode all the different ways doc.text() does - content = content_bytes.decode("utf-8") - except Exception as err: - log(f"Blobstore Error: Failed to read string from {kind}:{name}: {repr(err)}") + if not settings.ENABLE_BLOBSTORAGE: + return "" + try: + content = decode_document_content(retrieve_bytes(kind, name)) + except Exception as err: + log(f"Blobstore Error: Failed to read string from {kind}:{name}: {repr(err)}") + raise return content + + +def force_replication(kind: str, name: str): + if not settings.ENABLE_BLOBSTORAGE: + return + storage = _get_storage(kind) + from ietf.blobdb.storage import BlobdbStorage + if isinstance(storage, BlobdbStorage): + storage.force_replication(name) diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py index e24c58e1e7..37c235b911 100644 --- a/ietf/doc/tasks.py +++ b/ietf/doc/tasks.py @@ -1,17 +1,21 @@ -# Copyright The IETF Trust 2024, All Rights Reserved +# Copyright The IETF Trust 2024-2026, All Rights Reserved # # Celery task definitions # import datetime + import debug # pyflakes:ignore from celery import shared_task +from celery.exceptions import MaxRetriesExceededError from pathlib import Path from django.conf import settings from django.utils import timezone -from ietf.utils import log +from ietf.doc.utils_r2 import rfcs_are_in_r2 +from ietf.doc.utils_red import trigger_red_precomputer +from ietf.utils import log, searchindex from ietf.utils.timezone import datetime_today from .expire import ( @@ -29,10 +33,13 @@ from .utils import ( generate_idnits2_rfc_status, generate_idnits2_rfcs_obsoleted, + rebuild_reference_relations, update_or_create_draft_bibxml_file, ensure_draft_bibxml_path_exists, investigate_fragment, ) +from .utils_bofreq import fixup_bofreq_timestamps +from .utils_errata import signal_update_rfc_metadata @shared_task @@ -74,17 +81,19 @@ def expire_last_calls_task(): try: expire_last_call(doc) except Exception: - log.log(f"ERROR: Failed to expire last call for {doc.file_tag()} (id={doc.pk})") + log.log( + f"ERROR: Failed to expire last call for {doc.file_tag()} (id={doc.pk})" + ) else: log.log(f"Expired last call for {doc.file_tag()} (id={doc.pk})") -@shared_task +@shared_task def generate_idnits2_rfc_status_task(): outpath = Path(settings.DERIVED_DIR) / "idnits2-rfc-status" blob = generate_idnits2_rfc_status() try: - outpath.write_text(blob, encoding="utf8") # TODO-BLOBSTORE + outpath.write_text(blob, encoding="utf8") # TODO-BLOBSTORE except Exception as e: log.log(f"failed to write idnits2-rfc-status: {e}") @@ -94,7 +103,7 @@ def generate_idnits2_rfcs_obsoleted_task(): outpath = Path(settings.DERIVED_DIR) / "idnits2-rfcs-obsoleted" blob = generate_idnits2_rfcs_obsoleted() try: - outpath.write_text(blob, encoding="utf8") # TODO-BLOBSTORE + outpath.write_text(blob, encoding="utf8") # TODO-BLOBSTORE except Exception as e: log.log(f"failed to write idnits2-rfcs-obsoleted: {e}") @@ -102,7 +111,7 @@ def generate_idnits2_rfcs_obsoleted_task(): @shared_task def generate_draft_bibxml_files_task(days=7, process_all=False): """Generate bibxml files for recently updated docs - + If process_all is False (the default), processes only docs with new revisions in the last specified number of days. """ @@ -114,7 +123,9 @@ def generate_draft_bibxml_files_task(days=7, process_all=False): doc__type_id="draft", ).order_by("time") if not process_all: - doc_events = doc_events.filter(time__gte=timezone.now() - datetime.timedelta(days=days)) + doc_events = doc_events.filter( + time__gte=timezone.now() - datetime.timedelta(days=days) + ) for event in doc_events: try: update_or_create_draft_bibxml_file(event.doc, event.rev) @@ -128,3 +139,88 @@ def investigate_fragment_task(name_fragment: str): "name_fragment": name_fragment, "results": investigate_fragment(name_fragment), } + + +@shared_task +def rebuild_reference_relations_task(doc_names: list[str]): + log.log(f"Task: Rebuilding reference relations for {doc_names}") + for doc in Document.objects.filter(name__in=doc_names, type__in=["rfc", "draft"]): + filenames = dict() + base = ( + settings.RFC_PATH + if doc.type_id == "rfc" + else settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR + ) + stem = doc.name if doc.type_id == "rfc" else f"{doc.name}-{doc.rev}" + for ext in ["xml", "txt"]: + path = Path(base) / f"{stem}.{ext}" + if path.is_file(): + filenames[ext] = str(path) + if len(filenames) > 0: + rebuild_reference_relations(doc, filenames) + else: + log.log(f"Found no content for {stem}") + + +@shared_task +def fixup_bofreq_timestamps_task(): # pragma: nocover + fixup_bofreq_timestamps() + + +@shared_task +def signal_update_rfc_metadata_task(rfc_number_list=()): + signal_update_rfc_metadata(rfc_number_list) + + +@shared_task(bind=True) +def trigger_red_precomputer_task(self, rfc_number_list=()): + if not rfcs_are_in_r2(rfc_number_list): + log.log(f"Objects are not yet in R2 for RFCs {rfc_number_list}") + try: + countdown = getattr(settings, "RED_PRECOMPUTER_TRIGGER_RETRY_DELAY", 10) + max_retries = getattr(settings, "RED_PRECOMPUTER_TRIGGER_MAX_RETRIES", 12) + self.retry(countdown=countdown, max_retries=max_retries) + except MaxRetriesExceededError: + log.log(f"Gave up waiting for objects in R2 for RFCs {rfc_number_list}") + else: + trigger_red_precomputer(rfc_number_list) + + +@shared_task(bind=True) +def update_rfc_searchindex_task(self, rfc_number: int): + """Update the search index for one RFC""" + if not searchindex.enabled(): + log.log("Search indexing is not enabled, skipping") + return + + rfc = Document.objects.filter(type_id="rfc", rfc_number=rfc_number).first() + if rfc is None: + log.log( + f"ERROR: Document for rfc{rfc_number} not found, not updating search index" + ) + return + try: + searchindex.update_or_create_rfc_entry(rfc) + except Exception as err: + log.log(f"Search index update for {rfc.name} failed ({err})") + if isinstance(err, searchindex.RETRYABLE_ERROR_CLASSES): + searchindex_settings = searchindex.get_settings() + self.retry( + countdown=searchindex_settings["TASK_RETRY_DELAY"], + max_retries=searchindex_settings["TASK_MAX_RETRIES"], + ) + + +@shared_task +def rebuild_searchindex_task( + *, batchsize=40, drop_collection=False, upsert_presets=True +): + if drop_collection: + searchindex.delete_collection() + searchindex.create_collection() + if upsert_presets: + searchindex.upsert_presets() # ok if they already exist + searchindex.update_or_create_rfc_entries( + Document.objects.filter(type_id="rfc").order_by("-rfc_number"), + batchsize=batchsize, + ) diff --git a/ietf/doc/templatetags/ballot_icon.py b/ietf/doc/templatetags/ballot_icon.py index a94c145007..07a6c7f926 100644 --- a/ietf/doc/templatetags/ballot_icon.py +++ b/ietf/doc/templatetags/ballot_icon.py @@ -196,7 +196,7 @@ def state_age_colored(doc): .time ) except IndexError: - state_datetime = datetime.datetime(1990, 1, 1, tzinfo=datetime.timezone.utc) + state_datetime = datetime.datetime(1990, 1, 1, tzinfo=datetime.UTC) days = (timezone.now() - state_datetime).days # loosely based on the Publish Path page at the iesg wiki if iesg_state == "lc": diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py index d4adf96a27..ae5df641c2 100644 --- a/ietf/doc/templatetags/ietf_filters.py +++ b/ietf/doc/templatetags/ietf_filters.py @@ -285,7 +285,7 @@ def urlize_related_source_list(related, document_html=False): url=url) )) return links - + @register.filter(name='urlize_related_target_list', is_safe=True, document_html=False) def urlize_related_target_list(related, document_html=False): """Convert a list of RelatedDocuments into list of links using the target document's canonical name""" @@ -302,7 +302,7 @@ def urlize_related_target_list(related, document_html=False): url=url) )) return links - + @register.filter(name='dashify') def dashify(string): """ @@ -521,10 +521,52 @@ def plural(text, seq, arg='s'): else: return text + pluralize(len(seq), arg) + +# Translation table to escape ICS characters. The {} | {} construction builds up a dict +# mapping characters to arbitrary-length strings or None. Values in later dicts override +# earlier ones prior to conversion to a translation table, so excluding a char and then +# mapping it to an escape sequence results in its being escaped, not dropped. +rfc5545_text_escapes = str.maketrans( + # text = *(TSAFE-CHAR / ":" / DQUOTE / ESCAPED-CHAR) + # TSAFE-CHAR = WSP / %x21 / %x23-2B / %x2D-39 / %x3C-5B / + # %x5D-7E / NON-US-ASCII + {chr(c): None for c in range(0x00, 0x20)} # strip 0x00-0x20 + | { + # ESCAPED-CHAR = ("\\" / "\;" / "\," / "\N" / "\n") + "\n": r"\n", + ";": r"\;", + ",": r"\,", + "\\": r"\\", # rhs is two backslashes! + "\t": "\t", # htab ok (0x09) + " ": " ", # space ok (0x20) + } +) + + @register.filter def ics_esc(text): - text = re.sub(r"([\n,;\\])", r"\\\1", text) - return text + """Escape a string to use in an iCalendar text context + + >>> ics_esc('simple') + 'simple' + + For the next tests, it helps to know: + chr(0x09) = "\t" + chr(0x0a) = "\n" + chr(0x0d) = "\r" + chr(0x5c) = "\\" + + >>> ics_esc(f'strips{chr(0x0d)}out{chr(0x0d)}LFs') + 'stripsoutLFs' + + + >>> ics_esc(f'escapes;and,and{chr(0x5c)}and{chr(0x0a)}') + 'escapes\\\\;and\\\\,and\\\\\\\\and\\\\n' + + >>> ics_esc(f"keeps spaces : and{chr(0x09)}tabs") + 'keeps spaces : and\\ttabs' + """ + return text.translate(rfc5545_text_escapes) @register.simple_tag @@ -557,7 +599,7 @@ def ics_date_time(dt, tzname): return f':{timestamp}Z' else: return f';TZID={ics_esc(tzname)}:{timestamp}' - + @register.filter def next_day(value): return value + datetime.timedelta(days=1) @@ -676,7 +718,7 @@ def rfcbis(s): @stringfilter def urlize(value): raise RuntimeError("Use linkify from textfilters instead of urlize") - + @register.filter @stringfilter def charter_major_rev(rev): @@ -975,3 +1017,61 @@ def is_in_stream(doc): elif stream == "editorial": return True return False + + +@register.filter +def is_doc_ietf_adoptable(doc): + return doc.stream_id is None or all( + [ + doc.stream_id == "ietf", + doc.get_state_slug("draft-stream-ietf") + not in [ + "c-adopt", + "adopt-wg", + "info", + "wg-doc", + "parked", + "dead", + "wg-lc", + "waiting-for-implementation", + "chair-w", + "writeupw", + "sub-pub", + ], + doc.get_state_slug("draft") != "rfc", + doc.became_rfc() is None, + ] + ) + + +@register.filter +def can_issue_ietf_wg_lc(doc): + return all( + [ + doc.stream_id == "ietf", + doc.get_state_slug("draft-stream-ietf") + not in ["wg-cand", "c-adopt", "wg-lc"], + doc.get_state_slug("draft") != "rfc", + doc.became_rfc() is None, + ] + ) + + +@register.filter +def can_submit_to_iesg(doc): + return all( + [ + doc.stream_id == "ietf", + doc.get_state_slug("draft-iesg") == "idexists", + doc.get_state_slug("draft-stream-ietf") not in ["wg-cand", "c-adopt"], + ] + ) + + +@register.filter +def has_had_ietf_wg_lc(doc): + return ( + doc.stream_id == "ietf" + and doc.docevent_set.filter(statedocevent__state__slug="wg-lc").exists() + ) + diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index d74688f3f6..ff4461d466 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -10,7 +10,7 @@ from django.http import HttpRequest import lxml import bibtexparser -import mock +from unittest import mock import json import copy import random @@ -39,11 +39,15 @@ from ietf.doc.models import ( Document, DocRelationshipName, RelatedDocument, State, DocEvent, BallotPositionDocEvent, LastCallDocEvent, WriteupDocEvent, NewRevisionDocEvent, BallotType, EditedAuthorsDocEvent, StateType) -from ietf.doc.factories import ( DocumentFactory, DocEventFactory, CharterFactory, - ConflictReviewFactory, WgDraftFactory, IndividualDraftFactory, WgRfcFactory, - IndividualRfcFactory, StateDocEventFactory, BallotPositionDocEventFactory, - BallotDocEventFactory, DocumentAuthorFactory, NewRevisionDocEventFactory, - StatusChangeFactory, DocExtResourceFactory, RgDraftFactory, BcpFactory) +from ietf.doc.factories import (DocumentFactory, DocEventFactory, CharterFactory, + ConflictReviewFactory, WgDraftFactory, + IndividualDraftFactory, WgRfcFactory, + IndividualRfcFactory, StateDocEventFactory, + BallotPositionDocEventFactory, + BallotDocEventFactory, DocumentAuthorFactory, + NewRevisionDocEventFactory, + StatusChangeFactory, DocExtResourceFactory, + RgDraftFactory, BcpFactory, RfcAuthorFactory) from ietf.doc.forms import NotifyForm from ietf.doc.fields import SearchableDocumentsField from ietf.doc.utils import ( @@ -66,7 +70,7 @@ from ietf.name.models import SessionStatusName, BallotPositionName, DocTypeName, RoleName from ietf.person.models import Person from ietf.person.factories import PersonFactory, EmailFactory -from ietf.utils.mail import outbox, empty_outbox +from ietf.utils.mail import get_payload_text, outbox, empty_outbox from ietf.utils.test_utils import login_testing_unauthorized, unicontent from ietf.utils.test_utils import TestCase from ietf.utils.text import normalize_text @@ -403,6 +407,30 @@ def test_docs_for_ad(self): self.assertContains(r, discuss_other.doc.name) self.assertContains(r, block_other.doc.name) + def test_docs_for_iesg(self): + ad1 = RoleFactory(name_id='ad',group__type_id='area',group__state_id='active').person + ad2 = RoleFactory(name_id='ad',group__type_id='area',group__state_id='active').person + + draft = IndividualDraftFactory(ad=ad1) + draft.action_holders.set([PersonFactory()]) + draft.set_state(State.objects.get(type='draft-iesg', slug='lc')) + rfc = IndividualRfcFactory(ad=ad2) + conflrev = DocumentFactory(type_id='conflrev',ad=ad1) + conflrev.set_state(State.objects.get(type='conflrev', slug='iesgeval')) + statchg = DocumentFactory(type_id='statchg',ad=ad2) + statchg.set_state(State.objects.get(type='statchg', slug='iesgeval')) + charter = CharterFactory(name='charter-ietf-ames',ad=ad1) + charter.set_state(State.objects.get(type='charter', slug='iesgrev')) + + r = self.client.get(urlreverse('ietf.doc.views_search.docs_for_iesg')) + self.assertEqual(r.status_code, 200) + self.assertContains(r, draft.name) + self.assertContains(r, escape(draft.action_holders.first().name)) + self.assertNotContains(r, rfc.name) + self.assertContains(r, conflrev.name) + self.assertContains(r, statchg.name) + self.assertContains(r, charter.name) + def test_auth48_doc_for_ad(self): """Docs in AUTH48 state should have a decoration""" ad = RoleFactory(name_id='ad', group__type_id='area', group__state_id='active').person @@ -425,17 +453,6 @@ def test_drafts_in_last_call(self): self.assertContains(r, draft.title) self.assertContains(r, escape(draft.action_holders.first().name)) - def test_in_iesg_process(self): - doc_in_process = IndividualDraftFactory() - doc_in_process.action_holders.set([PersonFactory()]) - doc_in_process.set_state(State.objects.get(type='draft-iesg', slug='lc')) - doc_not_in_process = IndividualDraftFactory() - r = self.client.get(urlreverse('ietf.doc.views_search.drafts_in_iesg_process')) - self.assertEqual(r.status_code, 200) - self.assertContains(r, doc_in_process.title) - self.assertContains(r, escape(doc_in_process.action_holders.first().name)) - self.assertNotContains(r, doc_not_in_process.title) - def test_indexes(self): draft = IndividualDraftFactory() rfc = WgRfcFactory() @@ -966,7 +983,7 @@ def test_edit_authors_permissions(self): # Relevant users not authorized to edit authors unauthorized_usernames = [ 'plain', - *[author.user.username for author in draft.authors()], + *[author.user.username for author in draft.author_persons()], draft.group.get_chair().person.user.username, 'ad' ] @@ -981,7 +998,7 @@ def test_edit_authors_permissions(self): self.client.logout() # Try to add an author via POST - still only the secretary should be able to do this. - orig_authors = draft.authors() + orig_authors = draft.author_persons() post_data = self.make_edit_authors_post_data( basis='permission test', authors=draft.documentauthor_set.all(), @@ -999,12 +1016,40 @@ def test_edit_authors_permissions(self): for username in unauthorized_usernames: login_testing_unauthorized(self, username, url, method='post', request_kwargs=dict(data=post_data)) draft = Document.objects.get(pk=draft.pk) - self.assertEqual(draft.authors(), orig_authors) # ensure draft author list was not modified + self.assertEqual(draft.author_persons(), orig_authors) # ensure draft author list was not modified login_testing_unauthorized(self, 'secretary', url, method='post', request_kwargs=dict(data=post_data)) r = self.client.post(url, post_data) self.assertEqual(r.status_code, 302) draft = Document.objects.get(pk=draft.pk) - self.assertEqual(draft.authors(), orig_authors + [new_auth_person]) + self.assertEqual(draft.author_persons(), orig_authors + [new_auth_person]) + + def test_edit_authors_blocked_when_rfcauthors_exist(self): + """edit_authors returns 403 for all users when RfcAuthors exist""" + rfc = WgRfcFactory() + RfcAuthorFactory(document=rfc) + url = urlreverse('ietf.doc.views_doc.edit_authors', kwargs=dict(name=rfc.name)) + + self.client.login(username='secretary', password='secretary+password') + r = self.client.get(url) + self.assertEqual(r.status_code, 403) + r = self.client.post(url, {}) + self.assertEqual(r.status_code, 403) + + def test_document_main_hides_edit_authors_when_rfcauthors_exist(self): + """document_main does not offer edit link for authors when RfcAuthors exist""" + rfc = WgRfcFactory() + edit_authors_url = urlreverse('ietf.doc.views_doc.edit_authors', kwargs=dict(name=rfc.name)) + + self.client.login(username='secretary', password='secretary+password') + + r = self.client.get(urlreverse('ietf.doc.views_doc.document_main', kwargs=dict(name=rfc.name))) + self.assertEqual(r.status_code, 200) + self.assertContains(r, edit_authors_url) + + RfcAuthorFactory(document=rfc) + r = self.client.get(urlreverse('ietf.doc.views_doc.document_main', kwargs=dict(name=rfc.name))) + self.assertEqual(r.status_code, 200) + self.assertNotContains(r, edit_authors_url) def make_edit_authors_post_data(self, basis, authors): """Helper to generate edit_authors POST data for a set of authors""" @@ -1352,8 +1397,8 @@ def test_edit_authors_edit_fields(self): basis=change_reason ) - old_address = draft.authors()[0].email() - new_email = EmailFactory(person=draft.authors()[0], address=f'changed-{old_address}') + old_address = draft.author_persons()[0].email() + new_email = EmailFactory(person=draft.author_persons()[0], address=f'changed-{old_address}') post_data['author-0-email'] = new_email.address post_data['author-1-affiliation'] = 'University of Nowhere' post_data['author-2-country'] = 'Chile' @@ -1386,17 +1431,17 @@ def test_edit_authors_edit_fields(self): country_event = change_events.filter(desc__icontains='changed country').first() self.assertIsNotNone(email_event) - self.assertIn(draft.authors()[0].name, email_event.desc) + self.assertIn(draft.author_persons()[0].name, email_event.desc) self.assertIn(before[0]['email'], email_event.desc) self.assertIn(after[0]['email'], email_event.desc) self.assertIsNotNone(affiliation_event) - self.assertIn(draft.authors()[1].name, affiliation_event.desc) + self.assertIn(draft.author_persons()[1].name, affiliation_event.desc) self.assertIn(before[1]['affiliation'], affiliation_event.desc) self.assertIn(after[1]['affiliation'], affiliation_event.desc) self.assertIsNotNone(country_event) - self.assertIn(draft.authors()[2].name, country_event.desc) + self.assertIn(draft.author_persons()[2].name, country_event.desc) self.assertIn(before[2]['country'], country_event.desc) self.assertIn(after[2]['country'], country_event.desc) @@ -1850,13 +1895,63 @@ def test_document_ballot_needed_positions(self): def test_document_json(self): doc = IndividualDraftFactory() - + author = DocumentAuthorFactory(document=doc) + r = self.client.get(urlreverse("ietf.doc.views_doc.document_json", kwargs=dict(name=doc.name))) self.assertEqual(r.status_code, 200) data = r.json() - self.assertEqual(doc.name, data['name']) - self.assertEqual(doc.pages,data['pages']) + self.assertEqual(data["name"], doc.name) + self.assertEqual(data["pages"], doc.pages) + self.assertEqual( + data["authors"], + [ + { + "name": author.person.name, + "email": author.email.address, + "affiliation": author.affiliation, + } + ] + ) + + def test_document_json_rfc(self): + doc = IndividualRfcFactory() + old_style_author = DocumentAuthorFactory(document=doc) + url = urlreverse("ietf.doc.views_doc.document_json", kwargs=dict(name=doc.name)) + + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + data = r.json() + self.assertEqual(data["name"], doc.name) + self.assertEqual(data["pages"], doc.pages) + self.assertEqual( + data["authors"], + [ + { + "name": old_style_author.person.name, + "email": old_style_author.email.address, + "affiliation": old_style_author.affiliation, + } + ] + ) + + new_style_author = RfcAuthorFactory(document=doc) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + data = r.json() + self.assertEqual(data["name"], doc.name) + self.assertEqual(data["pages"], doc.pages) + self.assertEqual( + data["authors"], + [ + { + "name": new_style_author.titlepage_name, + "email": new_style_author.email.address, + "affiliation": new_style_author.affiliation, + } + ] + ) + def test_writeup(self): doc = IndividualDraftFactory(states = [('draft','active'),('draft-iesg','iesg-eva')],) @@ -1998,9 +2093,9 @@ def test_rfc_feed(self): self.assertEqual(len(q("item")), 3) item = q("item")[0] media_content = item.findall("{http://search.yahoo.com/mrss/}content") - self.assertEqual(len(media_content), 3) + self.assertEqual(len(media_content), 2) types = set([m.attrib["type"] for m in media_content]) - self.assertEqual(types, set(["text/plain", "text/html", "application/pdf"])) + self.assertEqual(types, set(["text/plain", "text/html"])) def test_state_help(self): url = urlreverse('ietf.doc.views_help.state_help', kwargs=dict(type="draft-iesg")) @@ -2148,20 +2243,19 @@ def test_trailing_hypen_digit_name_bibxml(self): class AddCommentTestCase(TestCase): def test_add_comment(self): - draft = WgDraftFactory(name='draft-ietf-mars-test',group__acronym='mars') - url = urlreverse('ietf.doc.views_doc.add_comment', kwargs=dict(name=draft.name)) + draft = WgDraftFactory(name="draft-ietf-mars-test", group__acronym="mars") + url = urlreverse("ietf.doc.views_doc.add_comment", kwargs=dict(name=draft.name)) login_testing_unauthorized(self, "secretary", url) # normal get r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(unicontent(r)) - self.assertEqual(len(q('form textarea[name=comment]')), 1) + self.assertEqual(len(q("form textarea[name=comment]")), 1) - # request resurrect events_before = draft.docevent_set.count() mailbox_before = len(outbox) - + r = self.client.post(url, dict(comment="This is a test.")) self.assertEqual(r.status_code, 302) @@ -2169,9 +2263,9 @@ def test_add_comment(self): self.assertEqual("This is a test.", draft.latest_event().desc) self.assertEqual("added_comment", draft.latest_event().type) self.assertEqual(len(outbox), mailbox_before + 1) - self.assertIn("Comment added", outbox[-1]['Subject']) - self.assertIn(draft.name, outbox[-1]['Subject']) - self.assertIn('draft-ietf-mars-test@', outbox[-1]['To']) + self.assertIn("Comment added", outbox[-1]["Subject"]) + self.assertIn(draft.name, outbox[-1]["Subject"]) + self.assertIn("draft-ietf-mars-test@", outbox[-1]["To"]) # Make sure we can also do it as IANA self.client.login(username="iana", password="iana+password") @@ -2180,7 +2274,22 @@ def test_add_comment(self): r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(unicontent(r)) - self.assertEqual(len(q('form textarea[name=comment]')), 1) + self.assertEqual(len(q("form textarea[name=comment]")), 1) + + empty_outbox() + rfc = WgRfcFactory() + self.client.login(username="rfc", password="rfc+password") + url = urlreverse("ietf.doc.views_doc.add_comment", kwargs=dict(name=rfc.name)) + r = self.client.post( + url, dict(comment="This is an RFC Editor comment on an RFC.") + ) + self.assertEqual(r.status_code, 302) + + self.assertEqual( + "This is an RFC Editor comment on an RFC.", rfc.latest_event().desc + ) + self.assertEqual(len(outbox), 1) + self.assertIn("This is an RFC Editor comment on an RFC.", get_payload_text(outbox[0])) class TemplateTagTest(TestCase): diff --git a/ietf/doc/tests_ballot.py b/ietf/doc/tests_ballot.py index ec23f3d491..517a2ce056 100644 --- a/ietf/doc/tests_ballot.py +++ b/ietf/doc/tests_ballot.py @@ -3,7 +3,6 @@ import datetime -import mock from pyquery import PyQuery @@ -25,7 +24,6 @@ from ietf.group.models import Group, Role from ietf.group.factories import GroupFactory, RoleFactory, ReviewTeamFactory from ietf.ipr.factories import HolderIprDisclosureFactory -from ietf.name.models import BallotPositionName from ietf.iesg.models import TelechatDate from ietf.person.models import Person from ietf.person.factories import PersonFactory, PersonalApiKeyFactory @@ -37,9 +35,18 @@ class EditPositionTests(TestCase): + + # N.B. This test needs to be rewritten to exercise all types of ballots (iesg, irsg, rsab) + # and test against the output of the mailtriggers instead of looking for hardcoded values + # in the To and CC results. See #7864 def test_edit_position(self): ad = Person.objects.get(user__username="ad") - draft = IndividualDraftFactory(ad=ad,stream_id='ietf') + draft = WgDraftFactory( + ad=ad, + stream_id="ietf", + notify="somebody@example.com", + group__acronym="mars", + ) ballot = create_ballot_if_not_open(None, draft, ad, 'approve') url = urlreverse('ietf.doc.views_ballot.edit_position', kwargs=dict(name=draft.name, ballot_id=ballot.pk)) @@ -55,11 +62,20 @@ def test_edit_position(self): self.assertEqual(len(q('form textarea[name=comment]')), 1) # vote + empty_outbox() events_before = draft.docevent_set.count() - - r = self.client.post(url, dict(position="discuss", - discuss=" This is a discussion test. \n ", - comment=" This is a test. \n ")) + + r = self.client.post( + url, + dict( + position="discuss", + discuss=" This is a discussion test. \n ", + comment=" This is a test. \n ", + additional_cc="test298347@example.com", + cc_choices=["doc_notify", "doc_group_chairs"], + send_mail=1, + ), + ) self.assertEqual(r.status_code, 302) pos = draft.latest_event(BallotPositionDocEvent, balloter=ad) @@ -70,6 +86,22 @@ def test_edit_position(self): self.assertTrue(pos.comment_time != None) self.assertTrue("New position" in pos.desc) self.assertEqual(draft.docevent_set.count(), events_before + 3) + self.assertEqual(len(outbox),1) + m = outbox[0] + self.assertTrue("COMMENT" in m['Subject']) + self.assertTrue("DISCUSS" in m['Subject']) + self.assertTrue(draft.name in m['Subject']) + self.assertTrue("This is a discussion test." in str(m)) + self.assertTrue("This is a test" in str(m)) + self.assertTrue("iesg@" in m['To']) + # cc_choice doc_group_chairs + self.assertTrue("mars-chairs@" in m['Cc']) + # cc_choice doc_notify + self.assertTrue("somebody@example.com" in m['Cc']) + # cc_choice doc_group_email_list was not selected + self.assertFalse(draft.group.list_email in m['Cc']) + # extra-cc + self.assertTrue("test298347@example.com" in m['Cc']) # recast vote events_before = draft.docevent_set.count() @@ -230,64 +262,6 @@ def test_cannot_edit_position_as_pre_ad(self): r = self.client.post(url, dict(position="discuss", discuss="Test discuss text")) self.assertEqual(r.status_code, 403) - # N.B. This test needs to be rewritten to exercise all types of ballots (iesg, irsg, rsab) - # and test against the output of the mailtriggers instead of looking for hardcoded values - # in the To and CC results. See #7864 - def test_send_ballot_comment(self): - ad = Person.objects.get(user__username="ad") - draft = WgDraftFactory(ad=ad,group__acronym='mars') - draft.notify = "somebody@example.com" - draft.save_with_history([DocEvent.objects.create(doc=draft, rev=draft.rev, type="changed_document", by=Person.objects.get(user__username="secretary"), desc="Test")]) - - ballot = create_ballot_if_not_open(None, draft, ad, 'approve') - - BallotPositionDocEvent.objects.create( - doc=draft, rev=draft.rev, type="changed_ballot_position", - by=ad, balloter=ad, ballot=ballot, pos=BallotPositionName.objects.get(slug="discuss"), - discuss="This draft seems to be lacking a clearer title?", - discuss_time=timezone.now(), - comment="Test!", - comment_time=timezone.now()) - - url = urlreverse('ietf.doc.views_ballot.send_ballot_comment', kwargs=dict(name=draft.name, - ballot_id=ballot.pk)) - login_testing_unauthorized(self, "ad", url) - - # normal get - r = self.client.get(url) - self.assertEqual(r.status_code, 200) - q = PyQuery(r.content) - self.assertTrue(len(q('form input[name="extra_cc"]')) > 0) - - # send - mailbox_before = len(outbox) - - r = self.client.post(url, dict(extra_cc="test298347@example.com", cc_choices=['doc_notify','doc_group_chairs'])) - self.assertEqual(r.status_code, 302) - - self.assertEqual(len(outbox), mailbox_before + 1) - m = outbox[-1] - self.assertTrue("COMMENT" in m['Subject']) - self.assertTrue("DISCUSS" in m['Subject']) - self.assertTrue(draft.name in m['Subject']) - self.assertTrue("clearer title" in str(m)) - self.assertTrue("Test!" in str(m)) - self.assertTrue("iesg@" in m['To']) - # cc_choice doc_group_chairs - self.assertTrue("mars-chairs@" in m['Cc']) - # cc_choice doc_notify - self.assertTrue("somebody@example.com" in m['Cc']) - # cc_choice doc_group_email_list was not selected - self.assertFalse(draft.group.list_email in m['Cc']) - # extra-cc - self.assertTrue("test298347@example.com" in m['Cc']) - - r = self.client.post(url, dict(cc="")) - self.assertEqual(r.status_code, 302) - self.assertEqual(len(outbox), mailbox_before + 2) - m = outbox[-1] - self.assertTrue("iesg@" in m['To']) - self.assertFalse(m['Cc'] and draft.group.list_email in m['Cc']) class BallotWriteupsTests(TestCase): @@ -741,11 +715,8 @@ def verify_can_see(username, url): verify_can_see(username, url) class ApproveBallotTests(TestCase): - @mock.patch('ietf.sync.rfceditor.requests.post', autospec=True) - def test_approve_ballot(self, mock_urlopen): - mock_urlopen.return_value.text = b'OK' - mock_urlopen.return_value.status_code = 200 - # + def test_approve_ballot(self): + ad = Person.objects.get(name="Areað Irector") draft = IndividualDraftFactory(ad=ad, intended_std_level_id='ps') draft.set_state(State.objects.get(used=True, type="draft-iesg", slug="iesg-eva")) # make sure it's approvable diff --git a/ietf/doc/tests_bofreq.py b/ietf/doc/tests_bofreq.py index 6a7c9393ef..6b142149be 100644 --- a/ietf/doc/tests_bofreq.py +++ b/ietf/doc/tests_bofreq.py @@ -307,17 +307,20 @@ def test_submit(self): url = urlreverse('ietf.doc.views_bofreq.submit', kwargs=dict(name=doc.name)) rev = doc.rev + doc_time = doc.time r = self.client.post(url,{'bofreq_submission':'enter','bofreq_content':'# oiwefrase'}) self.assertEqual(r.status_code, 302) doc = reload_db_objects(doc) - self.assertEqual(rev, doc.rev) + self.assertEqual(doc.rev, rev) + self.assertEqual(doc.time, doc_time) nobody = PersonFactory() self.client.login(username=nobody.user.username, password=nobody.user.username+'+password') r = self.client.post(url,{'bofreq_submission':'enter','bofreq_content':'# oiwefrase'}) self.assertEqual(r.status_code, 403) doc = reload_db_objects(doc) - self.assertEqual(rev, doc.rev) + self.assertEqual(doc.rev, rev) + self.assertEqual(doc.time, doc_time) self.client.logout() editor = bofreq_editors(doc).first() @@ -339,12 +342,14 @@ def test_submit(self): r = self.client.post(url, postdict) self.assertEqual(r.status_code, 302) doc = reload_db_objects(doc) - self.assertEqual('%02d'%(int(rev)+1) ,doc.rev) - self.assertEqual(f'# {username}', doc.text()) - self.assertEqual(f'# {username}', retrieve_str('bofreq',doc.get_base_name())) - self.assertEqual(docevent_count+1, doc.docevent_set.count()) - self.assertEqual(1, len(outbox)) + self.assertEqual(doc.rev, '%02d'%(int(rev)+1)) + self.assertGreater(doc.time, doc_time) + self.assertEqual(doc.text(), f'# {username}') + self.assertEqual(retrieve_str('bofreq', doc.get_base_name()), f'# {username}') + self.assertEqual(doc.docevent_set.count(), docevent_count+1) + self.assertEqual(len(outbox), 1) rev = doc.rev + doc_time = doc.time finally: os.unlink(file.name) diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py index 4753c4ff0c..db9dbc2baf 100644 --- a/ietf/doc/tests_draft.py +++ b/ietf/doc/tests_draft.py @@ -2,16 +2,16 @@ # -*- coding: utf-8 -*- +import json import os import datetime import io -import mock from collections import Counter from pathlib import Path from pyquery import PyQuery -from django.db.models import Q +from django.db.models import Max, Q from django.urls import reverse as urlreverse from django.conf import settings from django.utils import timezone @@ -20,13 +20,14 @@ import debug # pyflakes:ignore from ietf.doc.expire import expirable_drafts, get_expired_drafts, send_expire_notice_for_draft, expire_draft -from ietf.doc.factories import EditorialDraftFactory, IndividualDraftFactory, WgDraftFactory, RgDraftFactory, DocEventFactory +from ietf.doc.factories import EditorialDraftFactory, IndividualDraftFactory, StateDocEventFactory, WgDraftFactory, RgDraftFactory, DocEventFactory, WgRfcFactory from ietf.doc.models import ( Document, DocReminder, DocEvent, ConsensusDocEvent, LastCallDocEvent, RelatedDocument, State, TelechatDocEvent, WriteupDocEvent, DocRelationshipName, IanaExpertDocEvent ) from ietf.doc.storage_utils import exists_in_storage, store_str from ietf.doc.utils import get_tags_for_stream_id, create_ballot_if_not_open -from ietf.doc.views_draft import AdoptDraftForm +from ietf.doc.views_draft import AdoptDraftForm, IssueCallForAdoptionForm, IssueWorkingGroupLastCallForm +from ietf.ietfauth.utils import has_role from ietf.name.models import DocTagName, RoleName from ietf.group.factories import GroupFactory, RoleFactory from ietf.group.models import Group, Role @@ -85,7 +86,7 @@ def test_ad_approved(self): self.assertTrue("Approved: " in outbox[-1]['Subject']) self.assertTrue(draft.name in outbox[-1]['Subject']) self.assertTrue('iesg@' in outbox[-1]['To']) - + def test_change_state(self): ad = Person.objects.get(user__username="ad") draft = WgDraftFactory( @@ -138,7 +139,7 @@ def test_change_state(self): self.assertEqual(draft.get_state_slug("draft-iesg"), "review-e") self.assertTrue(not draft.tags.filter(slug="ad-f-up")) self.assertTrue(draft.tags.filter(slug="need-rev")) - self.assertCountEqual(draft.action_holders.all(), [ad] + draft.authors()) + self.assertCountEqual(draft.action_holders.all(), [ad] + draft.author_persons()) self.assertEqual(draft.docevent_set.count(), events_before + 3) self.assertTrue("Test comment" in draft.docevent_set.all()[0].desc) self.assertTrue("Changed action holders" in draft.docevent_set.all()[1].desc) @@ -177,7 +178,7 @@ def test_pull_from_rfc_queue(self): states=[('draft-iesg','rfcqueue')], ) DocEventFactory(type='started_iesg_process',by=ad,doc=draft,rev=draft.rev,desc="Started IESG Process") - draft.action_holders.add(*(draft.authors())) + draft.action_holders.add(*(draft.author_persons())) url = urlreverse('ietf.doc.views_draft.change_state', kwargs=dict(name=draft.name)) login_testing_unauthorized(self, "secretary", url) @@ -277,7 +278,7 @@ def test_request_last_call(self): states=[('draft-iesg','ad-eval')], ) DocEventFactory(type='started_iesg_process',by=ad,doc=draft,rev=draft.rev,desc="Started IESG Process") - draft.action_holders.add(*(draft.authors())) + draft.action_holders.add(*(draft.author_persons())) self.client.login(username="secretary", password="secretary+password") url = urlreverse('ietf.doc.views_draft.change_state', kwargs=dict(name=draft.name)) @@ -678,11 +679,11 @@ def test_in_draft_expire_freeze(self): datetime.datetime.combine( ietf_monday - datetime.timedelta(days=1), datetime.time(0, 0, 0), - tzinfo=datetime.timezone.utc, + tzinfo=datetime.UTC, ) )) self.assertFalse(in_draft_expire_freeze( - datetime.datetime.combine(ietf_monday, datetime.time(0, 0, 0), tzinfo=datetime.timezone.utc) + datetime.datetime.combine(ietf_monday, datetime.time(0, 0, 0), tzinfo=datetime.UTC) )) def test_warn_expirable_drafts(self): @@ -1367,7 +1368,7 @@ def _test_changing_ah(action_holders, reason): _test_changing_ah([doc.ad, doc.shepherd.person], 'this is a first test') _test_changing_ah([doc.ad], 'this is a second test') - _test_changing_ah(doc.authors(), 'authors can do it, too') + _test_changing_ah(doc.author_persons(), 'authors can do it, too') _test_changing_ah([], 'clear it back out') def test_doc_change_action_holders_as_doc_manager(self): @@ -1547,11 +1548,8 @@ def test_confirm_submission_no_doc_ad(self): class RequestPublicationTests(TestCase): - @mock.patch('ietf.sync.rfceditor.requests.post', autospec=True) - def test_request_publication(self, mockobj): - mockobj.return_value.text = b'OK' - mockobj.return_value.status_code = 200 - # + def test_request_publication(self): + draft = IndividualDraftFactory(stream_id='iab',group__acronym='iab',intended_std_level_id='inf',states=[('draft-stream-iab','approved')]) url = urlreverse('ietf.doc.views_draft.request_publication', kwargs=dict(name=draft.name)) @@ -1708,10 +1706,7 @@ def test_adopt_document(self): self.assertEqual(draft.stream_id, stream_state_type_slug[type_id][13:]) # trim off "draft-stream-" self.assertEqual(draft.docevent_set.count() - events_before, 5) self.assertEqual(len(outbox), 1) - self.assertTrue("Call For Adoption" in outbox[-1]["Subject"]) - self.assertTrue(f"{chair_role.group.acronym}-chairs@" in outbox[-1]['To']) - self.assertTrue(f"{draft.name}@" in outbox[-1]['To']) - self.assertTrue(f"{chair_role.group.acronym}@" in outbox[-1]['To']) + # contents of outbox[1] are tested elsewhere # adopt empty_outbox() @@ -2001,6 +1996,344 @@ def test_set_state(self): self.assertTrue("mars-chairs@ietf.org" in outbox[0].as_string()) self.assertTrue("marsdelegate@ietf.org" in outbox[0].as_string()) + def test_set_stream_state_to_wglc(self): + def _form_presents_state_option(response, state): + q = PyQuery(response.content) + option = q(f"select#id_new_state option[value='{state.pk}']") + return len(option) != 0 + + doc = WgDraftFactory() + chair = RoleFactory(name_id="chair", group=doc.group).person + url = urlreverse( + "ietf.doc.views_draft.change_stream_state", + kwargs=dict(name=doc.name, state_type="draft-stream-ietf"), + ) + login_testing_unauthorized(self, chair.user.username, url) + r = self.client.get(url) + wglc_state = State.objects.get(type="draft-stream-ietf", slug="wg-lc") + doc.set_state(wglc_state) + StateDocEventFactory( + doc=doc, + state_type_id="draft-stream-ietf", + state=("draft-stream-ietf", "wg-lc"), + ) + self.assertEqual(doc.docevent_set.count(), 2) + r = self.client.get(url) + self.assertTrue(_form_presents_state_option(r, wglc_state)) + other_doc = WgDraftFactory() + self.client.logout() + url = urlreverse( + "ietf.doc.views_draft.change_stream_state", + kwargs=dict(name=other_doc.name, state_type="draft-stream-ietf"), + ) + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + self.assertTrue(_form_presents_state_option(r, wglc_state)) + + def test_wg_call_for_adoption_issued(self): + role = RoleFactory( + name_id="chair", + group__acronym="mars", + group__list_email="mars-wg@ietf.org", + person__user__username="marschairman", + person__name="WG Cháir Man", + ) + # First test the usual workflow through the manage adoption view + draft = IndividualDraftFactory() + url = urlreverse( + "ietf.doc.views_draft.adopt_draft", kwargs=dict(name=draft.name) + ) + login_testing_unauthorized(self, "marschairman", url) + empty_outbox() + call_issued = State.objects.get(type="draft-stream-ietf", slug="c-adopt") + r = self.client.post( + url, + dict( + comment="some comment", + group=role.group.pk, + newstate=call_issued.pk, + weeks="10", + ), + ) + self.assertEqual(r.status_code, 302) + self.assertEqual(len(outbox), 1) + # Test not entering a duration on the form + draft = IndividualDraftFactory() + url = urlreverse( + "ietf.doc.views_draft.adopt_draft", kwargs=dict(name=draft.name) + ) + empty_outbox() + call_issued = State.objects.get(type="draft-stream-ietf", slug="c-adopt") + r = self.client.post( + url, + dict( + comment="some comment", + group=role.group.pk, + newstate=call_issued.pk, + ), + ) + self.assertEqual(r.status_code, 302) + self.assertEqual(len(outbox), 1) + + # Test the less usual workflow of issuing a call for adoption + # of a document that's already in the ietf stream + draft = WgDraftFactory(group=role.group) + url = urlreverse( + "ietf.doc.views_draft.change_stream_state", + kwargs=dict(name=draft.name, state_type="draft-stream-ietf"), + ) + old_state = draft.get_state("draft-stream-%s" % draft.stream_id) + new_state = State.objects.get( + used=True, type="draft-stream-%s" % draft.stream_id, slug="c-adopt" + ) + self.assertNotEqual(old_state, new_state) + empty_outbox() + r = self.client.post( + url, + dict( + new_state=new_state.pk, + comment="some comment", + weeks="10", + tags=[ + t.pk + for t in draft.tags.filter( + slug__in=get_tags_for_stream_id(draft.stream_id) + ) + ], + ), + ) + self.assertEqual(r.status_code, 302) + self.assertEqual(len(outbox), 1) + draft = WgDraftFactory(group=role.group) + url = urlreverse( + "ietf.doc.views_draft.change_stream_state", + kwargs=dict(name=draft.name, state_type="draft-stream-ietf"), + ) + old_state = draft.get_state("draft-stream-%s" % draft.stream_id) + new_state = State.objects.get( + used=True, type="draft-stream-%s" % draft.stream_id, slug="c-adopt" + ) + self.assertNotEqual(old_state, new_state) + empty_outbox() + r = self.client.post( + url, + dict( + new_state=new_state.pk, + comment="some comment", + tags=[ + t.pk + for t in draft.tags.filter( + slug__in=get_tags_for_stream_id(draft.stream_id) + ) + ], + ), + ) + self.assertEqual(r.status_code, 302) + self.assertEqual(len(outbox), 1) + + def test_issue_wg_lc_form(self): + end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=1) + post = dict( + end_date=end_date, + to="foo@example.net, bar@example.com", + # Intentionally not passing cc + subject=f"garbage {end_date.isoformat()}", + body=f"garbage {end_date.isoformat()}", + ) + form = IssueWorkingGroupLastCallForm(post) + self.assertTrue(form.is_valid()) + post["end_date"] = date_today(DEADLINE_TZINFO) + form = IssueWorkingGroupLastCallForm(post) + self.assertFalse(form.is_valid()) + self.assertIn( + "End date must be later than today", + form.errors["end_date"], + "Form accepted a too-early date", + ) + post["end_date"] = end_date + datetime.timedelta(days=2) + form = IssueWorkingGroupLastCallForm(post) + self.assertFalse(form.is_valid()) + self.assertIn( + f"Last call end date ({post['end_date'].isoformat()}) not found in subject", + form.errors["subject"], + "form allowed subject without end_date", + ) + self.assertIn( + f"Last call end date ({post['end_date'].isoformat()}) not found in body", + form.errors["body"], + "form allowed body without end_date", + ) + + def test_issue_wg_lc(self): + def _assert_rejected(testcase, doc, person): + url = urlreverse( + "ietf.doc.views_draft.issue_wg_lc", kwargs=dict(name=doc.name) + ) + login_testing_unauthorized(testcase, person.user.username, url) + r = testcase.client.get(url) + testcase.assertEqual(r.status_code, 404) + testcase.client.logout() + + already_rfc = WgDraftFactory(states=[("draft", "rfc")]) + rfc_chair = RoleFactory(name_id="chair", group=already_rfc.group).person + _assert_rejected(self, already_rfc, rfc_chair) + rg_doc = RgDraftFactory() + rg_chair = RoleFactory(name_id="chair", group=rg_doc.group).person + _assert_rejected(self, rg_doc, rg_chair) + inwglc_doc = WgDraftFactory(states=[("draft-stream-ietf", "wg-lc")]) + inwglc_chair = RoleFactory(name_id="chair", group=inwglc_doc.group).person + _assert_rejected(self, inwglc_doc, inwglc_chair) + doc = WgDraftFactory() + chair = RoleFactory(name_id="chair", group=doc.group).person + url = urlreverse("ietf.doc.views_draft.issue_wg_lc", kwargs=dict(name=doc.name)) + login_testing_unauthorized(self, chair.user.username, url) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + postdict = dict() + postdict["end_date"] = q("input#id_end_date").attr("value") + postdict["to"] = q("input#id_to").attr("value") + ", extrato@example.org" + cc = q("input#id_cc").attr("value") + if cc is not None: + postdict["cc"] = cc + ", extracc@example.org" + else: + postdict["cc"] = "extracc@example.org" + postdict["subject"] = q("input#id_subject").attr("value") + " Extra Subject Words" + postdict["body"] = q("textarea#id_body").text() + "FGgqbQ$UNeXs" + empty_outbox() + r = self.client.post( + url, + postdict, + ) + self.assertEqual(r.status_code, 302) + self.assertEqual(doc.get_state_slug("draft-stream-ietf"), "wg-lc") + self.assertEqual(len(outbox), 2) + self.assertIn(f"{doc.group.acronym}@ietf.org", outbox[1]["To"]) + self.assertIn("extrato@example.org", outbox[1]["To"]) + self.assertIn("extracc@example.org", outbox[1]["Cc"]) + self.assertIn("Extra Subject Words", outbox[1]["Subject"]) + self.assertIn("WG Last Call", outbox[1]["Subject"]) + body = get_payload_text(outbox[1]) + self.assertIn("disclosure obligations", body) + self.assertIn("FGgqbQ$UNeXs", body) + + def test_issue_wg_call_for_adoption_form(self): + end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=1) + post = dict( + end_date=end_date, + to="foo@example.net, bar@example.com", + # Intentionally not passing cc + subject=f"garbage {end_date.isoformat()}", + body=f"garbage {end_date.isoformat()}", + ) + form = IssueCallForAdoptionForm(post) + self.assertTrue(form.is_valid()) + post["end_date"] = date_today(DEADLINE_TZINFO) + form = IssueCallForAdoptionForm(post) + self.assertFalse(form.is_valid()) + self.assertIn( + "End date must be later than today", + form.errors["end_date"], + "Form accepted a too-early date", + ) + post["end_date"] = end_date + datetime.timedelta(days=2) + form = IssueCallForAdoptionForm(post) + self.assertFalse(form.is_valid()) + self.assertIn( + f"Call for adoption end date ({post['end_date'].isoformat()}) not found in subject", + form.errors["subject"], + "form allowed subject without end_date", + ) + self.assertIn( + f"Call for adoption end date ({post['end_date'].isoformat()}) not found in body", + form.errors["body"], + "form allowed body without end_date", + ) + + def test_issue_wg_call_for_adoption(self): + def _assert_rejected(testcase, doc, person, group=None): + target_acronym = group.acronym if group is not None else doc.group.acronym + url = urlreverse( + "ietf.doc.views_draft.issue_wg_call_for_adoption", + kwargs=dict(name=doc.name, acronym=target_acronym), + ) + login_testing_unauthorized(testcase, person.user.username, url) + r = testcase.client.get(url) + testcase.assertEqual(r.status_code, 403) + testcase.client.logout() + + def _verify_call_issued(testcase, doc, chair_role): + url = urlreverse( + "ietf.doc.views_draft.issue_wg_call_for_adoption", + kwargs=dict(name=doc.name, acronym=chair_role.group.acronym), + ) + login_testing_unauthorized(testcase, chair_role.person.user.username, url) + r = testcase.client.get(url) + testcase.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + postdict = dict() + postdict["end_date"] = q("input#id_end_date").attr("value") + postdict["to"] = q("input#id_to").attr("value") + ", extrato@example.com" + self.assertIn(chair_role.group.list_email, postdict["to"]) + cc = q("input#id_cc").attr("value") + if cc is not None: + postdict["cc"] = cc + ", extracc@example.com" + else: + postdict["cc"] = "extracc@example.com" + postdict["subject"] = q("input#id_subject").attr("value") + " Extra Subject Words" + postdict["body"] = q("textarea#id_body").text() + "FGgqbQ$UNeXs" + empty_outbox() + r = testcase.client.post( + url, + postdict, + ) + testcase.assertEqual(r.status_code, 302) + doc.refresh_from_db() + self.assertEqual(doc.group, chair_role.group) + self.assertEqual(doc.get_state_slug("draft-stream-ietf"), "c-adopt") + self.assertEqual(len(outbox), 2) + self.assertIn(f"{doc.group.acronym}@ietf.org", outbox[1]["To"]) + self.assertIn("extrato@example.com", outbox[1]["To"]) + self.assertIn("extracc@example.com", outbox[1]["Cc"]) + self.assertIn("Call for adoption", outbox[1]["Subject"]) + self.assertIn("Extra Subject Words", outbox[1]["Subject"]) + body = get_payload_text(outbox[1]) + self.assertIn("disclosure obligations", body) + self.assertIn("FGgqbQ$UNeXs", body) + self.client.logout() + return doc + + already_rfc = WgDraftFactory(states=[("draft", "rfc")]) + rfc = WgRfcFactory(group=already_rfc.group) + already_rfc.relateddocument_set.create(relationship_id="became_rfc",target=rfc) + rfc_chair = RoleFactory(name_id="chair", group=already_rfc.group).person + _assert_rejected(self, already_rfc, rfc_chair) + rg_doc = RgDraftFactory() + rg_chair = RoleFactory(name_id="chair", group=rg_doc.group).person + _assert_rejected(self, rg_doc, rg_chair) + inwglc_doc = WgDraftFactory(states=[("draft-stream-ietf", "wg-lc")]) + inwglc_chair = RoleFactory(name_id="chair", group=inwglc_doc.group).person + _assert_rejected(self, inwglc_doc, inwglc_chair) + ind_doc = IndividualDraftFactory() + _assert_rejected(self, ind_doc, rg_chair, rg_doc.group) + + # Successful call issued for doc already in WG + doc = WgDraftFactory(states=[("draft-stream-ietf","wg-cand")]) + chair_role = RoleFactory(name_id="chair",group=doc.group) + _ = _verify_call_issued(self, doc, chair_role) + + # Successful call issued for doc not yet in WG + doc = IndividualDraftFactory() + chair_role = RoleFactory(name_id="chair",group__type_id="wg") + doc = _verify_call_issued(self, doc, chair_role) + self.assertEqual(doc.group, chair_role.group) + self.assertEqual(doc.stream_id, "ietf") + self.assertEqual(doc.get_state_slug("draft-stream-ietf"), "c-adopt") + self.assertCountEqual( + doc.docevent_set.values_list("type", flat=True), + ["changed_state", "changed_group", "changed_stream", "new_revision"] + ) + def test_pubreq_validation(self): role = RoleFactory(name_id='chair',group__acronym='mars',group__list_email='mars-wg@ietf.org',person__user__username='marschairman',person__name='WG Cháir Man') RoleFactory(name_id='delegate',group=role.group,person__user__email='marsdelegate@ietf.org') @@ -2196,3 +2529,259 @@ def test_editorial_metadata(self): top_level_metadata_headings = q("tbody>tr>th:first-child").text() self.assertNotIn("IESG", top_level_metadata_headings) self.assertNotIn("IANA", top_level_metadata_headings) + +class IetfGroupActionHelperTests(TestCase): + def test_manage_adoption_routing(self): + draft = IndividualDraftFactory() + nobody = PersonFactory() + rgchair = RoleFactory(group__type_id="rg", name_id="chair").person + wgchair = RoleFactory(group__type_id="wg", name_id="chair").person + multichair = RoleFactory(group__type_id="rg", name_id="chair").person + RoleFactory(group__type_id="wg", person=multichair, name_id="chair") + ad = RoleFactory(group__type_id="area", name_id="ad").person + secretary = Role.objects.filter( + name_id="secr", group__acronym="secretariat" + ).first() + self.assertIsNotNone(secretary) + secretary = secretary.person + self.assertFalse( + has_role(rgchair.user, ["Secretariat", "Area Director", "WG Chair"]) + ) + url = urlreverse( + "ietf.doc.views_doc.document_main", kwargs={"name": draft.name} + ) + ask_about_ietf_link = urlreverse( + "ietf.doc.views_draft.ask_about_ietf_adoption_call", + kwargs={"name": draft.name}, + ) + non_ietf_adoption_link = urlreverse( + "ietf.doc.views_draft.adopt_draft", kwargs={"name": draft.name} + ) + for person in (None, nobody, rgchair, wgchair, multichair, ad, secretary): + if person is not None: + self.client.login( + username=person.user.username, + password=f"{person.user.username}+password", + ) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + has_ask_about_ietf_link = len(q(f'a[href="{ask_about_ietf_link}"]')) != 0 + has_non_ietf_adoption_link = ( + len(q(f'a[href="{non_ietf_adoption_link}"]')) != 0 + ) + ask_about_r = self.client.get(ask_about_ietf_link) + ask_about_link_return_code = ask_about_r.status_code + if person == rgchair: + self.assertFalse(has_ask_about_ietf_link) + self.assertTrue(has_non_ietf_adoption_link) + self.assertEqual(ask_about_link_return_code, 403) + elif person in (ad, nobody, None): + self.assertFalse(has_ask_about_ietf_link) + self.assertFalse(has_non_ietf_adoption_link) + self.assertEqual( + ask_about_link_return_code, 302 if person is None else 403 + ) + else: + self.assertTrue(has_ask_about_ietf_link) + self.assertFalse(has_non_ietf_adoption_link) + self.assertEqual(ask_about_link_return_code, 200) + self.client.logout() + + def test_ask_about_ietf_adoption_call(self): + # Basic permission tests above + doc = IndividualDraftFactory() + self.assertEqual(doc.docevent_set.count(), 1) + chair_role = RoleFactory(group__type_id="wg", name_id="chair") + chair = chair_role.person + group = chair_role.group + othergroup = GroupFactory(type_id="wg") + url = urlreverse( + "ietf.doc.views_draft.ask_about_ietf_adoption_call", + kwargs={"name": doc.name}, + ) + login_testing_unauthorized(self, chair.user.username, url) + r = self.client.post(url, {"group": othergroup.pk}) + self.assertEqual(r.status_code, 200) + r = self.client.post(url, {"group": group.pk}) + self.assertEqual(r.status_code, 302) + + def test_offer_wg_action_helpers(self): + def _assert_view_presents_buttons(testcase, response, expected): + q = PyQuery(response.content) + for id, expect in expected: + button = q(f"#{id}") + testcase.assertEqual( + len(button) != 0, + expect + ) + + # View rejects access + came_from_draft = WgDraftFactory(states=[("draft","rfc")]) + rfc = WgRfcFactory(group=came_from_draft.group) + came_from_draft.relateddocument_set.create(relationship_id="became_rfc",target=rfc) + rfc_chair = RoleFactory(name_id="chair", group=rfc.group).person + url = urlreverse("ietf.doc.views_draft.offer_wg_action_helpers", kwargs=dict(name=came_from_draft.name)) + login_testing_unauthorized(self, rfc_chair.user.username, url) + r = self.client.get(url) + self.assertEqual(r.status_code, 404) + self.client.logout() + rg_draft = RgDraftFactory() + rg_chair = RoleFactory(group=rg_draft.group, name_id="chair").person + url = urlreverse("ietf.doc.views_draft.offer_wg_action_helpers", kwargs=dict(name=rg_draft.name)) + login_testing_unauthorized(self, rg_chair.user.username, url) + r = self.client.get(url) + self.assertEqual(r.status_code,404) + self.client.logout() + + # View offers access + draft = WgDraftFactory() + chair = RoleFactory(group=draft.group, name_id="chair").person + url = urlreverse("ietf.doc.views_draft.offer_wg_action_helpers", kwargs=dict(name=draft.name)) + login_testing_unauthorized(self, chair.user.username, url) + r = self.client.get(url) + self.assertEqual(r.status_code,200) + _assert_view_presents_buttons( + self, + r, + [ + ("id_wgadopt_button", False), + ("id_wglc_button", True), + ("id_pubreq_button", True), + ], + ) + draft.set_state(State.objects.get(type_id="draft-stream-ietf", slug="wg-cand")) + r = self.client.get(url) + self.assertEqual(r.status_code,200) + _assert_view_presents_buttons( + self, + r, + [ + ("id_wgadopt_button", True), + ("id_wglc_button", False), + ("id_pubreq_button", False), + ], + ) + draft.set_state(State.objects.get(type_id="draft-stream-ietf", slug="wg-lc")) + StateDocEventFactory( + doc=draft, + state_type_id="draft-stream-ietf", + state=("draft-stream-ietf", "wg-lc"), + ) + self.assertEqual(draft.docevent_set.count(), 2) + r = self.client.get(url) + self.assertEqual(r.status_code,200) + _assert_view_presents_buttons( + self, + r, + [ + ("id_wgadopt_button", False), + ("id_wglc_button", False), + ("id_pubreq_button", True), + ], + ) + draft.set_state(State.objects.get(type_id="draft-stream-ietf",slug="chair-w")) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + _assert_view_presents_buttons( + self, + r, + [ + ("id_wgadopt_button", False), + ("id_wglc_button", True), + ("id_pubreq_button", True), + ], + ) + self.assertContains(response=r,text="Issue Another Working Group Last Call", status_code=200) + other_draft = WgDraftFactory() + self.client.logout() + url = urlreverse("ietf.doc.views_draft.offer_wg_action_helpers", kwargs=dict(name=other_draft.name)) + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + _assert_view_presents_buttons( + self, + r, + [ + ("id_wgadopt_button", False), + ("id_wglc_button", True), + ("id_pubreq_button", True), + ], + ) + self.assertContains( + response=r, text="Issue Working Group Last Call", status_code=200 + ) + +class BallotEmailAjaxTests(TestCase): + def test_ajax_build_position_email(self): + def _post_json(self, url, json_to_post): + r = self.client.post( + url, json.dumps(json_to_post), content_type="application/json" + ) + self.assertEqual(r.status_code, 200) + return json.loads(r.content) + + doc = WgDraftFactory() + ad = RoleFactory( + name_id="ad", group=doc.group, person__name="Some Areadirector" + ).person + url = urlreverse("ietf.doc.views_ballot.ajax_build_position_email") + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + self.assertEqual(r.status_code, 405) + response = _post_json(self, url, {}) + self.assertFalse(response["success"]) + self.assertEqual(response["errors"], ["post_data not provided"]) + response = _post_json(self, url, {"dictis": "not empty"}) + self.assertFalse(response["success"]) + self.assertEqual(response["errors"], ["post_data not provided"]) + response = _post_json(self, url, {"post_data": {}}) + self.assertFalse(response["success"]) + self.assertEqual(len(response["errors"]), 7) + response = _post_json( + self, + url, + { + "post_data": { + "discuss": "aaaaaa", + "comment": "bbbbbb", + "position": "discuss", + "balloter": Person.objects.aggregate(maxpk=Max("pk") + 1)["maxpk"], + "docname": "this-draft-does-not-exist", + "cc_choices": ["doc_group_mail_list"], + "additional_cc": "foo@example.com", + } + }, + ) + self.assertFalse(response["success"]) + self.assertEqual( + response["errors"], + ["No person found matching balloter", "No document found matching docname"], + ) + response = _post_json( + self, + url, + { + "post_data": { + "discuss": "aaaaaa", + "comment": "bbbbbb", + "position": "discuss", + "balloter": ad.pk, + "docname": doc.name, + "cc_choices": ["doc_group_mail_list"], + "additional_cc": "foo@example.com", + } + }, + ) + self.assertTrue(response["success"]) + for snippet in [ + "aaaaaa", + "bbbbbb", + "DISCUSS", + ad.plain_name(), + doc.name, + doc.group.list_email, + "foo@example.com", + ]: + self.assertIn(snippet, response["text"]) + diff --git a/ietf/doc/tests_irsg_ballot.py b/ietf/doc/tests_irsg_ballot.py index aa62d8aaf9..d96cf9dbef 100644 --- a/ietf/doc/tests_irsg_ballot.py +++ b/ietf/doc/tests_irsg_ballot.py @@ -355,28 +355,35 @@ def test_issue_ballot(self): def test_take_and_email_position(self): draft = RgDraftFactory() ballot = IRSGBallotDocEventFactory(doc=draft) - url = urlreverse('ietf.doc.views_ballot.edit_position', kwargs=dict(name=draft.name, ballot_id=ballot.pk)) + self.balloter + url = ( + urlreverse( + "ietf.doc.views_ballot.edit_position", + kwargs=dict(name=draft.name, ballot_id=ballot.pk), + ) + + self.balloter + ) empty_outbox() login_testing_unauthorized(self, self.username, url) r = self.client.get(url) self.assertEqual(r.status_code, 200) - r = self.client.post(url, dict(position='yes', comment='oib239sb', send_mail='Save and send email')) + empty_outbox() + r = self.client.post( + url, + dict( + position="yes", + comment="oib239sb", + send_mail="Save and send email", + cc_choices=["doc_authors", "doc_group_chairs", "doc_group_mail_list"], + ), + ) self.assertEqual(r.status_code, 302) e = draft.latest_event(BallotPositionDocEvent) - self.assertEqual(e.pos.slug,'yes') - self.assertEqual(e.comment, 'oib239sb') - - url = urlreverse('ietf.doc.views_ballot.send_ballot_comment', kwargs=dict(name=draft.name, ballot_id=ballot.pk)) + self.balloter - - r = self.client.get(url) - self.assertEqual(r.status_code, 200) - - r = self.client.post(url, dict(cc_choices=['doc_authors','doc_group_chairs','doc_group_mail_list'], body="Stuff")) - self.assertEqual(r.status_code, 302) - self.assertEqual(len(outbox),1) - self.assertNotIn('discuss-criteria', get_payload_text(outbox[0])) + self.assertEqual(e.pos.slug, "yes") + self.assertEqual(e.comment, "oib239sb") + self.assertEqual(len(outbox), 1) + self.assertNotIn("discuss-criteria", get_payload_text(outbox[0])) def test_close_ballot(self): draft = RgDraftFactory() @@ -482,27 +489,31 @@ def test_cant_take_position_on_iesg_ballot(self): def test_take_and_email_position(self): draft = RgDraftFactory() ballot = IRSGBallotDocEventFactory(doc=draft) - url = urlreverse('ietf.doc.views_ballot.edit_position', kwargs=dict(name=draft.name, ballot_id=ballot.pk)) + url = urlreverse( + "ietf.doc.views_ballot.edit_position", + kwargs=dict(name=draft.name, ballot_id=ballot.pk), + ) empty_outbox() login_testing_unauthorized(self, self.username, url) r = self.client.get(url) self.assertEqual(r.status_code, 200) - r = self.client.post(url, dict(position='yes', comment='oib239sb', send_mail='Save and send email')) + r = self.client.post( + url, + dict( + position="yes", + comment="oib239sb", + send_mail="Save and send email", + cc_choices=["doc_authors", "doc_group_chairs", "doc_group_mail_list"], + ), + ) self.assertEqual(r.status_code, 302) e = draft.latest_event(BallotPositionDocEvent) - self.assertEqual(e.pos.slug,'yes') - self.assertEqual(e.comment, 'oib239sb') - - url = urlreverse('ietf.doc.views_ballot.send_ballot_comment', kwargs=dict(name=draft.name, ballot_id=ballot.pk)) - - r = self.client.get(url) - self.assertEqual(r.status_code, 200) - - r = self.client.post(url, dict(cc_choices=['doc_authors','doc_group_chairs','doc_group_mail_list'], body="Stuff")) + self.assertEqual(e.pos.slug, "yes") + self.assertEqual(e.comment, "oib239sb") self.assertEqual(r.status_code, 302) - self.assertEqual(len(outbox),1) + self.assertEqual(len(outbox), 1) class IESGMemberTests(TestCase): diff --git a/ietf/doc/tests_material.py b/ietf/doc/tests_material.py index c87341c95b..04779bdaf1 100644 --- a/ietf/doc/tests_material.py +++ b/ietf/doc/tests_material.py @@ -6,7 +6,7 @@ import shutil import io -from mock import call, patch +from unittest.mock import call, patch from pathlib import Path from pyquery import PyQuery diff --git a/ietf/doc/tests_notprepped.py b/ietf/doc/tests_notprepped.py new file mode 100644 index 0000000000..f417aa7931 --- /dev/null +++ b/ietf/doc/tests_notprepped.py @@ -0,0 +1,122 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.conf import settings +from django.utils import timezone +from django.urls import reverse as urlreverse + +from pyquery import PyQuery + +from ietf.doc.factories import WgRfcFactory +from ietf.doc.models import StoredObject +from ietf.doc.storage_utils import store_bytes +from ietf.utils.test_utils import TestCase + + +class NotpreppedRfcXmlTests(TestCase): + def test_editor_source_button_visibility(self): + pre_v3 = WgRfcFactory(rfc_number=settings.FIRST_V3_RFC - 1) + first_v3 = WgRfcFactory(rfc_number=settings.FIRST_V3_RFC) + post_v3 = WgRfcFactory(rfc_number=settings.FIRST_V3_RFC + 1) + + for rfc, expect_button in [(pre_v3, False), (first_v3, True), (post_v3, True)]: + r = self.client.get( + urlreverse( + "ietf.doc.views_doc.document_main", kwargs=dict(name=rfc.name) + ) + ) + self.assertEqual(r.status_code, 200) + buttons = PyQuery(r.content)('a.btn:contains("Get editor source")') + if expect_button: + self.assertEqual(len(buttons), 1, msg=f"rfc_number={rfc.rfc_number}") + expected_href = urlreverse( + "ietf.doc.views_doc.rfcxml_notprepped_wrapper", + kwargs=dict(number=rfc.rfc_number), + ) + self.assertEqual( + buttons.attr("href"), + expected_href, + msg=f"rfc_number={rfc.rfc_number}", + ) + else: + self.assertEqual(len(buttons), 0, msg=f"rfc_number={rfc.rfc_number}") + + def test_rfcxml_notprepped(self): + number = settings.FIRST_V3_RFC + stored_name = f"notprepped/rfc{number}.notprepped.xml" + url = f"/doc/rfc{number}/notprepped/" + + # 404 for pre-v3 RFC numbers (no document needed) + r = self.client.get(f"/doc/rfc{number - 1}/notprepped/") + self.assertEqual(r.status_code, 404) + + # 404 when no RFC document exists in the database + r = self.client.get(url) + self.assertEqual(r.status_code, 404) + + # 404 when RFC document exists but has no StoredObject + WgRfcFactory(rfc_number=number) + r = self.client.get(url) + self.assertEqual(r.status_code, 404) + + # 404 when StoredObject exists but backing storage is missing (FileNotFoundError) + now = timezone.now() + StoredObject.objects.create( + store="rfc", + name=stored_name, + sha384="a" * 96, + len=0, + store_created=now, + created=now, + modified=now, + ) + r = self.client.get(url) + self.assertEqual(r.status_code, 404) + + # 200 with correct content-type, attachment disposition, and body when object is fully stored + xml_content = b"test" + store_bytes("rfc", stored_name, xml_content, allow_overwrite=True) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertEqual(r["Content-Type"], "application/xml") + self.assertEqual( + r["Content-Disposition"], + f'attachment; filename="rfc{number}.notprepped.xml"', + ) + self.assertEqual(b"".join(r.streaming_content), xml_content) + + def test_rfcxml_notprepped_wrapper(self): + number = settings.FIRST_V3_RFC + + # 404 for pre-v3 RFC numbers (no document needed) + r = self.client.get( + urlreverse( + "ietf.doc.views_doc.rfcxml_notprepped_wrapper", + kwargs=dict(number=number - 1), + ) + ) + self.assertEqual(r.status_code, 404) + + # 404 when no RFC document exists in the database + r = self.client.get( + urlreverse( + "ietf.doc.views_doc.rfcxml_notprepped_wrapper", + kwargs=dict(number=number), + ) + ) + self.assertEqual(r.status_code, 404) + + # 200 with rendered template when RFC document exists + rfc = WgRfcFactory(rfc_number=number) + r = self.client.get( + urlreverse( + "ietf.doc.views_doc.rfcxml_notprepped_wrapper", + kwargs=dict(number=number), + ) + ) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + self.assertIn(str(rfc.rfc_number), q("h1").text()) + download_url = urlreverse( + "ietf.doc.views_doc.rfcxml_notprepped", kwargs=dict(number=number) + ) + self.assertEqual(len(q(f'a.btn[href="{download_url}"]')), 1) diff --git a/ietf/doc/tests_review.py b/ietf/doc/tests_review.py index 13ddbc22ba..82d1b5c232 100644 --- a/ietf/doc/tests_review.py +++ b/ietf/doc/tests_review.py @@ -3,12 +3,12 @@ from pathlib import Path -import datetime, os, shutil +import datetime import io -import tarfile, tempfile, mailbox -import email.mime.multipart, email.mime.text, email.utils +import os +import shutil -from mock import patch +from unittest.mock import patch, Mock from requests import Response from django.apps import apps @@ -650,112 +650,132 @@ def test_accept_reviewer_assignment_after_reject(self): assignment = reload_db_objects(assignment) self.assertEqual(assignment.state_id, "accepted") - def make_test_mbox_tarball(self, review_req): - mbox_path = os.path.join(self.review_dir, "testmbox.tar.gz") - with tarfile.open(mbox_path, "w:gz") as tar: - with tempfile.NamedTemporaryFile(dir=self.review_dir, suffix=".mbox") as tmp: - mbox = mailbox.mbox(tmp.name) + @patch('ietf.review.mailarch.requests.post') + def test_retrieve_messages(self, mock_post): + mock_data = { + "results": [ + { + "from": "Alice ", + "subject": "Hello", + "content": "Hi\n This is a really good document.\n", + "message_id": "abc123", + "url": "https://example.com/message", + "date": "2025-04-07T12:00:00", + } + ] + } + mock_post.return_value.json.return_value = mock_data - # plain text - msg = email.mime.text.MIMEText("Hello,\n\nI have reviewed the document and did not find any problems.\n\nJohn Doe") - msg["From"] = "John Doe " - msg["To"] = review_req.team.list_email - msg["Subject"] = "Review of {}-01".format(review_req.doc.name) - msg["Message-ID"] = email.utils.make_msgid() - msg["Archived-At"] = "" - msg["Date"] = email.utils.formatdate() - - mbox.add(msg) - - # plain text + HTML - msg = email.mime.multipart.MIMEMultipart('alternative') - msg["From"] = "John Doe II " - msg["To"] = review_req.team.list_email - msg["Subject"] = "Review of {}".format(review_req.doc.name) - msg["Message-ID"] = email.utils.make_msgid() - msg["Archived-At"] = "" - - msg.attach(email.mime.text.MIMEText("Hi!,\r\nLooks OK!\r\n-John", "plain")) - msg.attach(email.mime.text.MIMEText("

Hi!,

Looks OK!

-John

", "html")) - mbox.add(msg) - - tmp.flush() - - tar.add(os.path.relpath(tmp.name)) + doc = WgDraftFactory(group__acronym='mars', rev='01') + review_team = ReviewTeamFactory(acronym="reviewteam", name="Review Team", type_id="review", list_email="reviewteam@ietf.org", parent=Group.objects.get(acronym="farfut")) + rev_role = RoleFactory(group=review_team, person__user__username='reviewer', person__user__email='reviewer@example.com', name_id='reviewer') + RoleFactory(group=review_team, person__user__username='reviewsecretary', person__user__email='reviewsecretary@example.com', name_id='secr') + review_req = ReviewRequestFactory(doc=doc, team=review_team, type_id='early', state_id='assigned', requested_by=rev_role.person, deadline=timezone.now() + datetime.timedelta(days=20)) + ReviewAssignmentFactory(review_request=review_req, reviewer=rev_role.person.email_set.first(), state_id='accepted') - mbox.close() + query_data = ietf.review.mailarch.construct_query_data(doc, review_team, query=None) + response = ietf.review.mailarch.retrieve_messages(query_data) - return mbox_path + self.assertEqual(len(response), 1) + self.assertEqual(response[0]['from'], 'Alice ') + self.assertEqual(response[0]['splitfrom'], ('Alice', 'alice@example.com')) + self.assertEqual(response[0]['subject'], 'Hello') + self.assertEqual(response[0]['content'], 'Hi\n This is a really good document.') + self.assertEqual(response[0]['message_id'], 'abc123') + self.assertEqual(response[0]['url'], 'https://example.com/message') + self.assertEqual(response[0]['utcdate'], ('2025-04-07', '12:00:00')) - def test_search_mail_archive(self): - doc = WgDraftFactory(group__acronym='mars',rev='01') + def test_construct_query_data(self): + doc = WgDraftFactory(group__acronym='mars', rev='01') review_team = ReviewTeamFactory(acronym="reviewteam", name="Review Team", type_id="review", list_email="reviewteam@ietf.org", parent=Group.objects.get(acronym="farfut")) - rev_role = RoleFactory(group=review_team,person__user__username='reviewer',person__user__email='reviewer@example.com',name_id='reviewer') - RoleFactory(group=review_team,person__user__username='reviewsecretary',person__user__email='reviewsecretary@example.com',name_id='secr') - review_req = ReviewRequestFactory(doc=doc,team=review_team,type_id='early',state_id='assigned',requested_by=rev_role.person,deadline=timezone.now()+datetime.timedelta(days=20)) + data = ietf.review.mailarch.construct_query_data(doc, review_team, query=None) + self.assertEqual(data['start_date'], (date_today() - datetime.timedelta(days=180)).isoformat()) + self.assertEqual(data['email_list'], 'reviewteam') + self.assertEqual(data['query_value'], doc.name) + self.assertEqual(data['query'], f'subject:({doc.name})') + self.assertEqual(data['limit'], '30') + + @patch('ietf.doc.views_review.requests.post') + def test_search_mail_archive(self, mock_post): + doc = WgDraftFactory(group__acronym='mars', rev='01') + review_team = ReviewTeamFactory(acronym="reviewteam", name="Review Team", type_id="review", list_email="reviewteam@ietf.org", parent=Group.objects.get(acronym="farfut")) + rev_role = RoleFactory(group=review_team, person__user__username='reviewer', person__user__email='reviewer@example.com', name_id='reviewer') + RoleFactory(group=review_team, person__user__username='reviewsecretary', person__user__email='reviewsecretary@example.com', name_id='secr') + review_req = ReviewRequestFactory(doc=doc, team=review_team, type_id='early', state_id='assigned', requested_by=rev_role.person, deadline=timezone.now() + datetime.timedelta(days=20)) assignment = ReviewAssignmentFactory(review_request=review_req, reviewer=rev_role.person.email_set.first(), state_id='accepted') + mock_data = { + "results": [ + { + "from": "Alice ", + "subject": "Review of {}-01".format(review_req.doc.name), + "content": "Hi\n This is a really good document.\n", + "message_id": "abc123", + "url": "https://example.com/message", + "date": "2025-04-07T12:00:00", + }, + { + "from": "Joe ", + "subject": "Review of {}".format(review_req.doc.name), + "content": "Hi\n I believe this is the best document.\n", + "message_id": "abc456", + "url": "https://example.com/message", + "date": "2025-04-07T12:00:00", + } + ] + } + response1 = Mock() + response1.json.return_value = mock_data + + response2 = Mock() + response2.json.return_value = mock_data + + response3 = Mock() + response3.json.return_value = {"results": []} + + mock_post.side_effect = [response1, response2, response3] + # test URL construction - query_urls = ietf.review.mailarch.construct_query_urls(doc, review_team) - self.assertTrue(review_req.doc.name in query_urls["query_data_url"]) + query_data = ietf.review.mailarch.construct_query_data(doc, review_team) + self.assertTrue(review_req.doc.name in query_data["query_value"]) - # test parsing - mbox_path = self.make_test_mbox_tarball(review_req) + url = urlreverse('ietf.doc.views_review.search_mail_archive', kwargs={"name": doc.name, "assignment_id": assignment.pk}) + url2 = urlreverse('ietf.doc.views_review.search_mail_archive', kwargs={"name": doc.name, "acronym": review_team.acronym}) + login_testing_unauthorized(self, "reviewsecretary", url) - try: - # mock URL generator and point it to local file - for this - # to work, the module (and not the function) must be - # imported in the view - real_fn = ietf.review.mailarch.construct_query_urls - ietf.review.mailarch.construct_query_urls = lambda doc, team, query=None: { "query_data_url": "file://" + os.path.abspath(mbox_path) } - url = urlreverse('ietf.doc.views_review.search_mail_archive', kwargs={ "name": doc.name, "assignment_id": assignment.pk }) - url2 = urlreverse('ietf.doc.views_review.search_mail_archive', kwargs={ "name": doc.name, "acronym": review_team.acronym }) - login_testing_unauthorized(self, "reviewsecretary", url) - - r = self.client.get(url) - self.assertEqual(r.status_code, 200) - messages = r.json()["messages"] - self.assertEqual(len(messages), 2) - - r = self.client.get(url2) - self.assertEqual(r.status_code, 200) - messages = r.json()["messages"] - self.assertEqual(len(messages), 2) - - today = date_today(datetime.timezone.utc) - - self.assertEqual(messages[0]["url"], "https://www.example.com/testmessage") - self.assertTrue("John Doe" in messages[0]["content"]) - self.assertEqual(messages[0]["subject"], "Review of {}-01".format(review_req.doc.name)) - self.assertEqual(messages[0]["revision_guess"], "01") - self.assertEqual(messages[0]["splitfrom"], ["John Doe", "johndoe@example.com"]) - self.assertEqual(messages[0]["utcdate"][0], today.isoformat()) - - self.assertEqual(messages[1]["url"], "https://www.example.com/testmessage2") - self.assertTrue("Looks OK" in messages[1]["content"]) - self.assertTrue("" not in messages[1]["content"]) - self.assertEqual(messages[1]["subject"], "Review of {}".format(review_req.doc.name)) - self.assertFalse('revision_guess' in messages[1]) - self.assertEqual(messages[1]["splitfrom"], ["John Doe II", "johndoe2@example.com"]) - self.assertEqual(messages[1]["utcdate"][0], "") - - - # Test failure to return mailarch results - no_result_path = os.path.join(self.review_dir, "mailarch_no_result.html") - with io.open(no_result_path, "w") as f: - f.write('Content-Type: text/html\n\n
No results found
') - ietf.review.mailarch.construct_query_urls = lambda doc, team, query=None: { "query_data_url": "file://" + os.path.abspath(no_result_path) } - - url = urlreverse('ietf.doc.views_review.search_mail_archive', kwargs={ "name": doc.name, "assignment_id": assignment.pk }) - - r = self.client.get(url) - self.assertEqual(r.status_code, 200) - result = r.json() - self.assertNotIn('messages', result) - self.assertIn('No results found', result['error']) - - finally: - ietf.review.mailarch.construct_query_urls = real_fn + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + messages = r.json()["messages"] + self.assertEqual(len(messages), 2) + + r = self.client.get(url2) + self.assertEqual(r.status_code, 200) + messages = r.json()["messages"] + self.assertEqual(len(messages), 2) + + self.assertEqual(messages[0]["url"], "https://example.com/message") + self.assertTrue("Hi" in messages[0]["content"]) + self.assertEqual(messages[0]["subject"], "Review of {}-01".format(review_req.doc.name)) + self.assertEqual(messages[0]["revision_guess"], "01") + self.assertEqual(messages[0]["splitfrom"], ["Alice", "alice@example.com"]) + self.assertEqual(messages[0]["utcdate"], ['2025-04-07', '12:00:00']) + + self.assertEqual(messages[1]["url"], "https://example.com/message") + self.assertTrue("Hi" in messages[1]["content"]) + self.assertTrue("" not in messages[1]["content"]) + self.assertEqual(messages[1]["subject"], "Review of {}".format(review_req.doc.name)) + self.assertFalse('revision_guess' in messages[1]) + self.assertEqual(messages[1]["splitfrom"], ["Joe", "joe@example.com"]) + self.assertEqual(messages[1]["utcdate"], ['2025-04-07', '12:00:00']) + + # Test failure to return mailarch results + url = urlreverse('ietf.doc.views_review.search_mail_archive', kwargs={"name": doc.name, "assignment_id": assignment.pk}) + + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + result = r.json() + self.assertNotIn('messages', result) + self.assertIn('No results found', result['error']) def test_submit_unsolicited_review_choose_team(self): doc = WgDraftFactory(group__acronym='mars', rev='01') @@ -802,7 +822,7 @@ def test_complete_review_upload_content(self): r = self.client.get(url) self.assertEqual(r.status_code, 200) self.assertContains(r, assignment.review_request.team.list_email) - for author in assignment.review_request.doc.authors(): + for author in assignment.review_request.doc.author_persons(): self.assertContains(r, author.formatted_email()) # faulty post diff --git a/ietf/doc/tests_rsab_ballot.py b/ietf/doc/tests_rsab_ballot.py index 028f548232..9086106ba9 100644 --- a/ietf/doc/tests_rsab_ballot.py +++ b/ietf/doc/tests_rsab_ballot.py @@ -333,34 +333,19 @@ def test_take_and_email_position(self): r = self.client.get(url) self.assertEqual(r.status_code, 200) - r = self.client.post( - url, - dict(position="yes", comment="oib239sb", send_mail="Save and send email"), - ) - self.assertEqual(r.status_code, 302) - e = draft.latest_event(BallotPositionDocEvent) - self.assertEqual(e.pos.slug, "yes") - self.assertEqual(e.comment, "oib239sb") - - url = ( - urlreverse( - "ietf.doc.views_ballot.send_ballot_comment", - kwargs=dict(name=draft.name, ballot_id=ballot.pk), - ) - + self.balloter - ) - - r = self.client.get(url) - self.assertEqual(r.status_code, 200) - r = self.client.post( url, dict( + position="yes", + comment="oib239sb", + send_mail="Save and send email", cc_choices=["doc_authors", "doc_group_chairs", "doc_group_mail_list"], - body="Stuff", ), ) self.assertEqual(r.status_code, 302) + e = draft.latest_event(BallotPositionDocEvent) + self.assertEqual(e.pos.slug, "yes") + self.assertEqual(e.comment, "oib239sb") self.assertEqual(len(outbox), 1) self.assertNotIn("discuss-criteria", get_payload_text(outbox[0])) @@ -532,31 +517,19 @@ def test_take_and_email_position(self): r = self.client.get(url) self.assertEqual(r.status_code, 200) - r = self.client.post( - url, - dict(position="yes", comment="oib239sb", send_mail="Save and send email"), - ) - self.assertEqual(r.status_code, 302) - e = draft.latest_event(BallotPositionDocEvent) - self.assertEqual(e.pos.slug, "yes") - self.assertEqual(e.comment, "oib239sb") - - url = urlreverse( - "ietf.doc.views_ballot.send_ballot_comment", - kwargs=dict(name=draft.name, ballot_id=ballot.pk), - ) - - r = self.client.get(url) - self.assertEqual(r.status_code, 200) - r = self.client.post( url, dict( + position="yes", + comment="oib239sb", + send_mail="Save and send email", cc_choices=["doc_authors", "doc_group_chairs", "doc_group_mail_list"], - body="Stuff", ), ) self.assertEqual(r.status_code, 302) + e = draft.latest_event(BallotPositionDocEvent) + self.assertEqual(e.pos.slug, "yes") + self.assertEqual(e.comment, "oib239sb") self.assertEqual(len(outbox), 1) diff --git a/ietf/doc/tests_statement.py b/ietf/doc/tests_statement.py index fea42b97d6..bb097b1828 100644 --- a/ietf/doc/tests_statement.py +++ b/ietf/doc/tests_statement.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2023, All Rights Reserved +# Copyright The IETF Trust 2023-2025, All Rights Reserved import debug # pyflakes:ignore @@ -372,3 +372,36 @@ def test_submit_non_markdown_formats(self): self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertTrue("Unexpected content" in q("#id_statement_file").next().text()) + + def test_change_statement_state(self): + statement = StatementFactory() # starts in "active" state + active_state = State.objects.get(type_id="statement", slug="active") + replaced_state = State.objects.get(type_id="statement", slug="replaced") + url = urlreverse( + "ietf.doc.views_statement.change_statement_state", + kwargs={"name": statement.name}, + ) + + events_before = statement.docevent_set.count() + login_testing_unauthorized(self, "secretary", url) + + r = self.client.get(url) + self.assertEqual(r.status_code,200) + + r = self.client.post(url, {"state": active_state.pk}, follow=True) + self.assertContains(r, "State not changed", status_code=200) + statement = Document.objects.get(pk=statement.pk) # bust the state cache + self.assertEqual(statement.get_state(), active_state) + + r = self.client.post(url, {"state": replaced_state.pk}, follow=True) + self.assertContains(r, "State changed to", status_code=200) + statement = Document.objects.get(pk=statement.pk) # bust the state cache + self.assertEqual(statement.get_state(), replaced_state) + + events_after = statement.docevent_set.count() + self.assertEqual(events_after, events_before + 1) + event = statement.docevent_set.first() + self.assertEqual(event.type, "changed_state") + self.assertEqual( + event.desc, "Statement State changed to Replaced from Active" + ) diff --git a/ietf/doc/tests_tasks.py b/ietf/doc/tests_tasks.py index 8a6ffa8be1..48db95d047 100644 --- a/ietf/doc/tests_tasks.py +++ b/ietf/doc/tests_tasks.py @@ -1,18 +1,20 @@ -# Copyright The IETF Trust 2024, All Rights Reserved +# Copyright The IETF Trust 2024-2026, All Rights Reserved -import debug # pyflakes:ignore import datetime -import mock +from unittest import mock from pathlib import Path +from celery.exceptions import Retry from django.conf import settings +from django.test.utils import override_settings from django.utils import timezone +from typesense import exceptions as typesense_exceptions from ietf.utils.test_utils import TestCase from ietf.utils.timezone import datetime_today -from .factories import DocumentFactory, NewRevisionDocEventFactory +from .factories import DocumentFactory, NewRevisionDocEventFactory, WgRfcFactory from .models import Document, NewRevisionDocEvent from .tasks import ( expire_ids_task, @@ -22,8 +24,11 @@ generate_idnits2_rfc_status_task, investigate_fragment_task, notify_expirations_task, + rebuild_searchindex_task, + update_rfc_searchindex_task, ) + class TaskTests(TestCase): @mock.patch("ietf.doc.tasks.in_draft_expire_freeze") @mock.patch("ietf.doc.tasks.get_expired_drafts") @@ -87,7 +92,7 @@ def test_expire_last_calls_task(self, mock_get_expired, mock_expire): self.assertEqual(mock_expire.call_args_list[0], mock.call(docs[0])) self.assertEqual(mock_expire.call_args_list[1], mock.call(docs[1])) self.assertEqual(mock_expire.call_args_list[2], mock.call(docs[2])) - + # Check that it runs even if exceptions occur mock_get_expired.reset_mock() mock_expire.reset_mock() @@ -111,9 +116,92 @@ def test_investigate_fragment_task(self): retval, {"name_fragment": "some fragment", "results": investigation_results} ) + @mock.patch("ietf.doc.tasks.searchindex.update_or_create_rfc_entry") + @mock.patch("ietf.doc.tasks.searchindex.enabled") + def test_update_rfc_searchindex_task( + self, mock_searchindex_enabled, mock_create_entry + ): + mock_searchindex_enabled.return_value = False + + self.assertFalse(Document.objects.filter(rfc_number=5073).exists()) + rfc = WgRfcFactory() + update_rfc_searchindex_task(rfc_number=5073) + self.assertFalse(mock_create_entry.called) + update_rfc_searchindex_task(rfc_number=rfc.rfc_number) + self.assertFalse(mock_create_entry.called) + + mock_searchindex_enabled.return_value = True + update_rfc_searchindex_task(rfc_number=5073) + self.assertFalse(mock_create_entry.called) + update_rfc_searchindex_task(rfc_number=rfc.rfc_number) + self.assertTrue(mock_create_entry.called) + + with override_settings(SEARCHINDEX_CONFIG={"TASK_MAX_RETRIES": 0}): + # Try a non-retryable error (there are others) + mock_create_entry.side_effect = typesense_exceptions.RequestMalformed + update_rfc_searchindex_task(rfc_number=rfc.rfc_number) # no retry + # Now what should be a retryable error + mock_create_entry.side_effect = typesense_exceptions.Timeout + with self.assertRaises(Retry): + update_rfc_searchindex_task(rfc_number=rfc.rfc_number) + + @mock.patch("ietf.doc.tasks.searchindex.update_or_create_rfc_entries") + @mock.patch("ietf.doc.tasks.searchindex.upsert_presets") + @mock.patch("ietf.doc.tasks.searchindex.create_collection") + @mock.patch("ietf.doc.tasks.searchindex.delete_collection") + def test_rebuild_searchindex_task( + self, mock_delete, mock_create, mock_presets, mock_update + ): + rfcs = WgRfcFactory.create_batch(10) + rebuild_searchindex_task() + self.assertFalse(mock_delete.called) + self.assertFalse(mock_create.called) + self.assertTrue(mock_presets.called) + self.assertTrue(mock_update.called) + self.assertQuerysetEqual( + mock_update.call_args.args[0], + sorted(rfcs, key=lambda doc: -doc.rfc_number), + ordered=True, + ) + + mock_delete.reset_mock() + mock_create.reset_mock() + mock_presets.reset_mock() + mock_update.reset_mock() + rebuild_searchindex_task(drop_collection=True) + self.assertTrue(mock_delete.called) + self.assertTrue(mock_create.called) + self.assertTrue(mock_presets.called) + self.assertTrue(mock_update.called) + self.assertQuerysetEqual( + mock_update.call_args.args[0], + sorted(rfcs, key=lambda doc: -doc.rfc_number), + ordered=True, + ) + + mock_delete.reset_mock() + mock_create.reset_mock() + mock_presets.reset_mock() + mock_update.reset_mock() + rebuild_searchindex_task( + drop_collection=True, batchsize=3, upsert_presets=False + ) + self.assertTrue(mock_delete.called) + self.assertTrue(mock_create.called) + self.assertFalse(mock_presets.called) + self.assertTrue(mock_update.called) + self.assertQuerysetEqual( + mock_update.call_args.args[0], + sorted(rfcs, key=lambda doc: -doc.rfc_number), + ordered=True, + ) + self.assertEqual(mock_update.call_args.kwargs["batchsize"], 3) + class Idnits2SupportTests(TestCase): - settings_temp_path_overrides = TestCase.settings_temp_path_overrides + ['DERIVED_DIR'] + settings_temp_path_overrides = TestCase.settings_temp_path_overrides + [ + "DERIVED_DIR" + ] @mock.patch("ietf.doc.tasks.generate_idnits2_rfcs_obsoleted") def test_generate_idnits2_rfcs_obsoleted_task(self, mock_generate): @@ -151,7 +239,9 @@ def setUp(self): ) # a couple that should always be ignored NewRevisionDocEventFactory( - time=now - datetime.timedelta(days=6), rev="09", doc__type_id="rfc" # not a draft + time=now - datetime.timedelta(days=6), + rev="09", + doc__type_id="rfc", # not a draft ) NewRevisionDocEventFactory( type="changed_document", # not a "new_revision" type @@ -164,7 +254,9 @@ def setUp(self): @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists") @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file") - def test_generate_bibxml_files_for_all_drafts_task(self, mock_create, mock_ensure_path): + def test_generate_bibxml_files_for_all_drafts_task( + self, mock_create, mock_ensure_path + ): generate_draft_bibxml_files_task(process_all=True) self.assertTrue(mock_ensure_path.called) self.assertCountEqual( @@ -193,12 +285,15 @@ def test_generate_bibxml_files_for_all_drafts_task(self, mock_create, mock_ensur @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists") @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file") - def test_generate_bibxml_files_for_recent_drafts_task(self, mock_create, mock_ensure_path): + def test_generate_bibxml_files_for_recent_drafts_task( + self, mock_create, mock_ensure_path + ): # default args - look back 7 days generate_draft_bibxml_files_task() self.assertTrue(mock_ensure_path.called) self.assertCountEqual( - mock_create.call_args_list, [mock.call(self.young_event.doc, self.young_event.rev)] + mock_create.call_args_list, + [mock.call(self.young_event.doc, self.young_event.rev)], ) mock_create.reset_mock() mock_ensure_path.reset_mock() @@ -223,7 +318,9 @@ def test_generate_bibxml_files_for_recent_drafts_task(self, mock_create, mock_en @mock.patch("ietf.doc.tasks.ensure_draft_bibxml_path_exists") @mock.patch("ietf.doc.tasks.update_or_create_draft_bibxml_file") - def test_generate_bibxml_files_for_recent_drafts_task_with_bad_value(self, mock_create, mock_ensure_path): + def test_generate_bibxml_files_for_recent_drafts_task_with_bad_value( + self, mock_create, mock_ensure_path + ): with self.assertRaises(ValueError): generate_draft_bibxml_files_task(days=0) self.assertFalse(mock_create.called) diff --git a/ietf/doc/tests_utils.py b/ietf/doc/tests_utils.py index f610fe3d76..ba672cd847 100644 --- a/ietf/doc/tests_utils.py +++ b/ietf/doc/tests_utils.py @@ -1,25 +1,36 @@ # Copyright The IETF Trust 2020, All Rights Reserved import datetime +from io import BytesIO + +import mock import debug # pyflakes:ignore +import requests from pathlib import Path from unittest.mock import call, patch from django.conf import settings +from django.core.files.storage import storages from django.db import IntegrityError from django.test.utils import override_settings from django.utils import timezone + +from ietf.doc.utils_r2 import rfcs_are_in_r2 +from ietf.doc.utils_red import trigger_red_precomputer from ietf.group.factories import GroupFactory, RoleFactory from ietf.name.models import DocTagName from ietf.person.factories import PersonFactory +from ietf.doc.factories import BallotPositionDocEventFactory from ietf.utils.test_utils import TestCase, name_of_file_containing, reload_db_objects from ietf.person.models import Person from ietf.doc.factories import DocumentFactory, WgRfcFactory, WgDraftFactory -from ietf.doc.models import State, DocumentActionHolder, DocumentAuthor +from ietf.doc.models import State, DocumentActionHolder, DocumentAuthor, StoredObject from ietf.doc.utils import (update_action_holders, add_state_change_event, update_documentauthors, fuzzy_find_documents, rebuild_reference_relations, build_file_urls, - ensure_draft_bibxml_path_exists, update_or_create_draft_bibxml_file) + ensure_draft_bibxml_path_exists, update_or_create_draft_bibxml_file, + last_ballot_doc_revision) +from ietf.doc.storage_utils import store_str from ietf.utils.draft import Draft, PlaintextDraft from ietf.utils.xmldraft import XMLDraft @@ -148,7 +159,7 @@ def test_update_action_holders_resets_age(self): doc = self.doc_in_iesg_state('pub-req') doc.action_holders.set([self.ad]) dah = doc.documentactionholder_set.get(person=self.ad) - dah.time_added = datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc) # arbitrary date in the past + dah.time_added = datetime.datetime(2020, 1, 1, tzinfo=datetime.UTC) # arbitrary date in the past dah.save() right_now = timezone.now() @@ -387,13 +398,13 @@ def test_requires_txt_or_xml(self): result = rebuild_reference_relations(self.doc, {}) self.assertCountEqual(result.keys(), ['errors']) self.assertEqual(len(result['errors']), 1) - self.assertIn('No Internet-Draft text available', result['errors'][0], + self.assertIn('No file available', result['errors'][0], 'Error should be reported if no Internet-Draft file is given') result = rebuild_reference_relations(self.doc, {'md': 'cant-do-this.md'}) self.assertCountEqual(result.keys(), ['errors']) self.assertEqual(len(result['errors']), 1) - self.assertIn('No Internet-Draft text available', result['errors'][0], + self.assertIn('No file available', result['errors'][0], 'Error should be reported if no XML or plaintext file is given') @patch.object(XMLDraft, 'get_refs') @@ -533,3 +544,156 @@ def test_update_draft_bibxml_file(self, mock): self.assertEqual(mock.call_count, 1) self.assertEqual(mock.call_args, call(doc, "26")) self.assertEqual(ref_path.read_text(), "This\nis\nmy\nbibxml") + + +class LastBallotDocRevisionTests(TestCase): + def test_last_ballot_doc_revision(self): + now = timezone.now() + ad = Person.objects.get(user__username="ad") + bpde_with_null_send_email = BallotPositionDocEventFactory( + time=now - datetime.timedelta(minutes=30), + send_email=None, + ) + ballot = bpde_with_null_send_email.ballot + BallotPositionDocEventFactory( + ballot=ballot, + balloter=ad, + pos_id='noobj', + comment='Commentary', + comment_time=timezone.now(), + send_email=None, + ) + doc = bpde_with_null_send_email.doc + rev = bpde_with_null_send_email.rev + nobody = PersonFactory() + self.assertIsNone(last_ballot_doc_revision(doc, nobody)) + self.assertEqual(rev, last_ballot_doc_revision(doc, ad)) + + +class UtilsRedTests(TestCase): + @mock.patch("ietf.doc.utils_red.log") + @mock.patch("ietf.doc.utils_red.requests.post") + def test_trigger_red_precomputer_not_configured(self, mock_post, mock_log): + with override_settings(): + try: + del settings.CUSTOM_SETTING_NAME + except AttributeError: + pass + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + self.assertEqual(mock_log.call_count, 1) + mock_args, _ = mock_log.call_args + self.assertEqual( + mock_args, + ("No URL configured for triggering red precompute multiple, skipping",), + ) + + mock_log.reset_mock() + with override_settings(TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL=None): + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + self.assertFalse(mock_post.called) + self.assertEqual(mock_log.call_count, 1) + mock_args, _ = mock_log.call_args + self.assertEqual( + mock_args, + ("No URL configured for triggering red precompute multiple, skipping",), + ) + + @override_settings( + TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL="urlbits", + ) + @mock.patch("ietf.doc.utils_red.log") + @mock.patch("ietf.doc.utils_red.requests.post", side_effect=requests.Timeout()) + def test_trigger_red_precomputer_swallows_timeout_exception( + self, mock_post, mock_log + ): + exception_raised = False + try: + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + except Exception: + exception_raised = True + self.assertFalse(exception_raised) + self.assertEqual(mock_log.call_count, 2) + # only checking the last log call + mock_args, _ = mock_log.call_args + self.assertEqual(len(mock_args), 1) + self.assertIn("POST request timed out", mock_args[0]) + + @override_settings( + TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL="urlbits", + ) + @mock.patch("ietf.doc.utils_red.requests.post", side_effect=Exception()) + def test_trigger_red_precomputer_does_not_swallow_too_much(self, mock_post): + exception_raised = False + try: + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + except Exception: + exception_raised = True + self.assertTrue(exception_raised) + + @override_settings( + TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL="urlbits", + DEFAULT_REQUESTS_TIMEOUT=314159265, + ) + @mock.patch("ietf.doc.utils_red.log") + @mock.patch("ietf.doc.utils_red.requests.post") + def test_trigger_red_precomputer(self, mock_post, mock_log): + mock_post.return_value = mock.Mock(status_code=200) + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + self.assertTrue(mock_post.called) + _, mock_kwargs = mock_post.call_args + self.assertIn("url", mock_kwargs) + self.assertEqual(mock_kwargs["url"], "urlbits") + self.assertIn("json", mock_kwargs) + self.assertEqual(mock_kwargs["json"], {"rfcs": "1,2,3"}) + self.assertIn("timeout", mock_kwargs) + self.assertEqual(mock_kwargs["timeout"], 314159265) + self.assertEqual(mock_log.call_count, 1) # Not testing the first info log value + mock_log.reset_mock() + mock_post.reset_mock() + mock_post.return_value = mock.Mock( + status_code=500, + ) + trigger_red_precomputer(rfc_number_list=[1, 2, 3]) + self.assertEqual(mock_log.call_count, 2) + mock_args, _ = mock_log.call_args + self.assertEqual(len(mock_args), 1) + expected = f"POST request failed for {settings.TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL} : status_code=500" + self.assertEqual(mock_args[0], expected) + + +class UtilsR2TestCase(TestCase): + def test_rfcs_are_in_r2(self): + rfcs = WgRfcFactory.create_batch(2) + rfc_name_list = [rfc.name for rfc in rfcs] + rfc_number_list = [rfc.rfc_number for rfc in rfcs] + r2_rfc_bucket = storages["r2-rfc"] + # Right now the various doc Factories do not populate any content + self.assertEqual( + StoredObject.objects.filter( + store="rfc", doc_name__in=rfc_name_list + ).count(), + 0, + ) + self.assertTrue(rfcs_are_in_r2(rfc_number_list=rfc_number_list)) + for rfc in rfcs: + store_str( + kind="rfc", + name=f"testartifact/{rfc.name}.testartifact", + content="", + doc_name=rfc.name, + doc_rev=None, + ) + self.assertEqual( + StoredObject.objects.filter( + store="rfc", doc_name__in=rfc_name_list + ).count(), + 2, + ) + self.assertFalse(rfcs_are_in_r2(rfc_number_list=rfc_number_list)) + r2_rfc_bucket.save(f"testartifact/{rfcs[0].name}.testartifact", BytesIO(b"")) + self.assertFalse(rfcs_are_in_r2(rfc_number_list=rfc_number_list)) + r2_rfc_bucket.save(f"testartifact/{rfcs[1].name}.testartifact", BytesIO(b"")) + self.assertTrue(rfcs_are_in_r2(rfc_number_list=rfc_number_list)) + + + diff --git a/ietf/doc/urls.py b/ietf/doc/urls.py index 6e2f03a3dc..0c13503b78 100644 --- a/ietf/doc/urls.py +++ b/ietf/doc/urls.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2009-2023, All Rights Reserved +# Copyright The IETF Trust 2009-2025, All Rights Reserved # -*- coding: utf-8 -*- # Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). # All rights reserved. Contact: Pasi Eronen @@ -53,12 +53,13 @@ url(r'^ad/?$', views_search.ad_workload), url(r'^ad/(?P[^/]+)/?$', views_search.docs_for_ad), url(r'^ad2/(?P[\w.-]+)/$', RedirectView.as_view(url='/doc/ad/%(name)s/', permanent=True)), + url(r'^for_iesg/?$', RedirectView.as_view(pattern_name='ietf.doc.views_search.docs_for_iesg', permanent=False)), url(r'^rfc-status-changes/?$', views_status_change.rfc_status_changes), url(r'^start-rfc-status-change/(?:%(name)s/)?$' % settings.URL_REGEXPS, views_status_change.start_rfc_status_change), url(r'^bof-requests/?$', views_bofreq.bof_requests), url(r'^bof-requests/new/$', views_bofreq.new_bof_request), url(r'^statement/new/$', views_statement.new_statement), - url(r'^iesg/?$', views_search.drafts_in_iesg_process), + url(r'^iesg/?$', views_search.docs_for_iesg), url(r'^email-aliases/?$', views_doc.email_aliases), url(r'^downref/?$', views_downref.downref_registry), url(r'^downref/add/?$', views_downref.downref_registry_add), @@ -74,7 +75,7 @@ # This block should really all be at the idealized docs.ietf.org service url(r'^html/(?Pbcp[0-9]+?)(\.txt|\.html)?/?$', RedirectView.as_view(url=settings.RFC_EDITOR_INFO_BASE_URL+"%(name)s", permanent=False)), url(r'^html/(?Pstd[0-9]+?)(\.txt|\.html)?/?$', RedirectView.as_view(url=settings.RFC_EDITOR_INFO_BASE_URL+"%(name)s", permanent=False)), - url(r'^html/%(name)s(?:-%(rev)s)?(\.txt|\.html)?/?$' % settings.URL_REGEXPS, views_doc.document_html), + url(r'^html/%(name)s(?:-(?P[0-9]{2}(-[0-9]{2})?))?(\.txt|\.html)?/?$' % settings.URL_REGEXPS, views_doc.document_html), url(r'^id/%(name)s(?:-%(rev)s)?(?:\.(?P(txt|html|xml)))?/?$' % settings.URL_REGEXPS, views_doc.document_raw_id), url(r'^pdf/%(name)s(?:-%(rev)s)?(?:\.(?P[a-z]+))?/?$' % settings.URL_REGEXPS, views_doc.document_pdfized), @@ -92,10 +93,14 @@ url(r'^ballots/irsg/$', views_ballot.irsg_ballot_status), url(r'^ballots/rsab/$', views_ballot.rsab_ballot_status), + url(r'^build-position-email/$', views_ballot.ajax_build_position_email), + url(r'^(?P(bcp|std|fyi))/?$', views_search.index_subseries), url(r'^%(name)s(?:/%(rev)s)?/$' % settings.URL_REGEXPS, views_doc.document_main), url(r'^%(name)s(?:/%(rev)s)?/bibtex/$' % settings.URL_REGEXPS, views_doc.document_bibtex), + url(r'^rfc(?P[0-9]+)/notprepped/$' , views_doc.rfcxml_notprepped), + url(r'^rfc(?P[0-9]+)/notprepped-wrapper/$', views_doc.rfcxml_notprepped_wrapper), url(r'^%(name)s(?:/%(rev)s)?/idnits2-state/$' % settings.URL_REGEXPS, views_doc.idnits2_state), url(r'^bibxml3/reference.I-D.%(name)s(?:-%(rev)s)?.xml$' % settings.URL_REGEXPS, views_doc.document_bibxml_ref), url(r'^bibxml3/%(name)s(?:-%(rev)s)?.xml$' % settings.URL_REGEXPS, views_doc.document_bibxml), @@ -110,7 +115,6 @@ url(r'^%(name)s/ballot/rsab/$' % settings.URL_REGEXPS, views_doc.document_rsab_ballot), url(r'^%(name)s/ballot/(?P[0-9]+)/$' % settings.URL_REGEXPS, views_doc.document_ballot), url(r'^%(name)s/ballot/(?P[0-9]+)/position/$' % settings.URL_REGEXPS, views_ballot.edit_position), - url(r'^%(name)s/ballot/(?P[0-9]+)/emailposition/$' % settings.URL_REGEXPS, views_ballot.send_ballot_comment), url(r'^%(name)s/(?:%(rev)s/)?doc.json$' % settings.URL_REGEXPS, views_doc.document_json), url(r'^%(name)s/ballotpopup/(?P[0-9]+)/$' % settings.URL_REGEXPS, views_doc.ballot_popup), url(r'^(?P[A-Za-z0-9._+-]+)/reviewrequest/', include("ietf.doc.urls_review")), @@ -123,6 +127,7 @@ url(r'^%(name)s/edit/info/$' % settings.URL_REGEXPS, views_draft.edit_info), url(r'^%(name)s/edit/requestresurrect/$' % settings.URL_REGEXPS, views_draft.request_resurrect), url(r'^%(name)s/edit/submit-to-iesg/$' % settings.URL_REGEXPS, views_draft.to_iesg), + url(r'^%(name)s/edit/issue-wg-lc/$' % settings.URL_REGEXPS, views_draft.issue_wg_lc), url(r'^%(name)s/edit/resurrect/$' % settings.URL_REGEXPS, views_draft.resurrect), url(r'^%(name)s/edit/addcomment/$' % settings.URL_REGEXPS, views_doc.add_comment), @@ -141,9 +146,14 @@ url(r'^%(name)s/edit/shepherdemail/$' % settings.URL_REGEXPS, views_draft.change_shepherd_email), url(r'^%(name)s/edit/shepherdwriteup/$' % settings.URL_REGEXPS, views_draft.edit_shepherd_writeup), url(r'^%(name)s/edit/requestpublication/$' % settings.URL_REGEXPS, views_draft.request_publication), + url(r'^%(name)s/edit/ask-about-ietf-adoption/$' % settings.URL_REGEXPS, views_draft.ask_about_ietf_adoption_call), url(r'^%(name)s/edit/adopt/$' % settings.URL_REGEXPS, views_draft.adopt_draft), + url(r'^%(name)s/edit/issue-wg-call-for-adoption/%(acronym)s/$' % settings.URL_REGEXPS, views_draft.issue_wg_call_for_adoption), + url(r'^%(name)s/edit/release/$' % settings.URL_REGEXPS, views_draft.release_draft), url(r'^%(name)s/edit/state/(?Pdraft-stream-[a-z]+)/$' % settings.URL_REGEXPS, views_draft.change_stream_state), + url(r'^%(name)s/edit/wg-action-helpers/$' % settings.URL_REGEXPS, views_draft.offer_wg_action_helpers), + url(r'^%(name)s/edit/state/statement/$' % settings.URL_REGEXPS, views_statement.change_statement_state), url(r'^%(name)s/edit/clearballot/(?P[\w-]+)/$' % settings.URL_REGEXPS, views_ballot.clear_ballot), url(r'^%(name)s/edit/deferballot/$' % settings.URL_REGEXPS, views_ballot.defer_ballot), diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py index 3ddd904c75..5f8f587c59 100644 --- a/ietf/doc/utils.py +++ b/ietf/doc/utils.py @@ -4,6 +4,7 @@ import datetime import io +import json import math import os import re @@ -13,7 +14,7 @@ from dataclasses import dataclass from hashlib import sha384 from pathlib import Path -from typing import Iterator, Optional, Union +from typing import Iterator, Optional, Union, Iterable from zoneinfo import ZoneInfo from django.conf import settings @@ -33,14 +34,25 @@ from ietf.community.models import CommunityList from ietf.community.utils import docs_tracked_by_community_list -from ietf.doc.models import Document, DocHistory, State, DocumentAuthor, DocHistoryAuthor +from ietf.doc.models import ( + DocHistory, + DocHistoryAuthor, + Document, + DocumentAuthor, + EditedRfcAuthorsDocEvent, + RfcAuthor, + State, + StoredObject, +) from ietf.doc.models import RelatedDocument, RelatedDocHistory, BallotType, DocReminder from ietf.doc.models import DocEvent, ConsensusDocEvent, BallotDocEvent, IRSGBallotDocEvent, NewRevisionDocEvent, StateDocEvent -from ietf.doc.models import TelechatDocEvent, DocumentActionHolder, EditedAuthorsDocEvent +from ietf.doc.models import TelechatDocEvent, DocumentActionHolder, EditedAuthorsDocEvent, BallotPositionDocEvent +from ietf.doc.storage_utils import force_replication from ietf.name.models import DocReminderTypeName, DocRelationshipName from ietf.group.models import Role, Group, GroupFeatures from ietf.ietfauth.utils import has_role, is_authorized_in_doc_stream, is_individual_draft_author, is_bofreq_editor from ietf.person.models import Email, Person +from ietf.person.utils import get_active_balloters from ietf.review.models import ReviewWish from ietf.utils import draft, log from ietf.utils.mail import parseaddr, send_mail @@ -533,7 +545,7 @@ def update_action_holders(doc, prev_state=None, new_state=None, prev_tags=None, doc.action_holders.clear() if tags.removed("need-rev"): # Removed the 'need-rev' tag - drop authors from the action holders list - DocumentActionHolder.objects.filter(document=doc, person__in=doc.authors()).delete() + DocumentActionHolder.objects.filter(document=doc, person__in=doc.author_persons()).delete() elif tags.added("need-rev"): # Remove the AD if we're asking for a new revision DocumentActionHolder.objects.filter(document=doc, person=doc.ad).delete() @@ -548,7 +560,7 @@ def update_action_holders(doc, prev_state=None, new_state=None, prev_tags=None, doc.action_holders.add(doc.ad) # Authors get the action if a revision is needed if tags.added("need-rev"): - for auth in doc.authors(): + for auth in doc.author_persons(): doc.action_holders.add(auth) # Now create an event if we changed the set @@ -560,6 +572,40 @@ def update_action_holders(doc, prev_state=None, new_state=None, prev_tags=None, ) +def _change_field_and_describe( + author: DocumentAuthor | RfcAuthor, + field: str, + newval, + field_display_name: str | None = None, +): + # make the change + oldval = getattr(author, field) + setattr(author, field, newval) + + was_empty = oldval is None or len(str(oldval)) == 0 + now_empty = newval is None or len(str(newval)) == 0 + + # describe the change + if oldval == newval: + return None + else: + if field_display_name is None: + field_display_name = field + + if was_empty and not now_empty: + return 'set {field} to "{new}"'.format( + field=field_display_name, new=newval + ) + elif now_empty and not was_empty: + return 'cleared {field} (was "{old}")'.format( + field=field_display_name, old=oldval + ) + else: + return 'changed {field} from "{old}" to "{new}"'.format( + field=field_display_name, old=oldval, new=newval + ) + + def update_documentauthors(doc, new_docauthors, by=None, basis=None): """Update the list of authors for a document @@ -572,27 +618,6 @@ def update_documentauthors(doc, new_docauthors, by=None, basis=None): used. These objects will not be saved, their attributes will be used to create new DocumentAuthor instances. (The document and order fields will be ignored.) """ - def _change_field_and_describe(auth, field, newval): - # make the change - oldval = getattr(auth, field) - setattr(auth, field, newval) - - was_empty = oldval is None or len(str(oldval)) == 0 - now_empty = newval is None or len(str(newval)) == 0 - - # describe the change - if oldval == newval: - return None - else: - if was_empty and not now_empty: - return 'set {field} to "{new}"'.format(field=field, new=newval) - elif now_empty and not was_empty: - return 'cleared {field} (was "{old}")'.format(field=field, old=oldval) - else: - return 'changed {field} from "{old}" to "{new}"'.format( - field=field, old=oldval, new=newval - ) - persons = [] changes = [] # list of change descriptions @@ -636,6 +661,123 @@ def _change_field_and_describe(auth, field, newval): ) for change in changes ] + +def update_rfcauthors( + rfc: Document, new_rfcauthors: Iterable[RfcAuthor], by: Person | None = None +) -> Iterable[EditedRfcAuthorsDocEvent]: + def _find_matching_author( + author_to_match: RfcAuthor, existing_authors: Iterable[RfcAuthor] + ) -> RfcAuthor | None: + """Helper to find a matching existing author""" + if author_to_match.person_id is not None: + for candidate in existing_authors: + if candidate.person_id == author_to_match.person_id: + return candidate + return None # no match + # author does not have a person, match on titlepage name + for candidate in existing_authors: + if candidate.titlepage_name == author_to_match.titlepage_name: + return candidate + return None # no match + + def _rfcauthor_from_documentauthor(docauthor: DocumentAuthor) -> RfcAuthor: + """Helper to create an equivalent RfcAuthor from a DocumentAuthor""" + return RfcAuthor( + document_id=docauthor.document_id, + titlepage_name=docauthor.person.plain_name(), # closest thing we have + is_editor=False, + person_id=docauthor.person_id, + affiliation=docauthor.affiliation, + country=docauthor.country, + order=docauthor.order, + ) + + # Is this the first time this document is getting an RfcAuthor? If so, the + # updates will need to account for the model change. + converting_from_docauthors = not rfc.rfcauthor_set.exists() + + if converting_from_docauthors: + original_authors = [ + _rfcauthor_from_documentauthor(da) for da in rfc.documentauthor_set.all() + ] + else: + original_authors = list(rfc.rfcauthor_set.all()) + + authors_to_commit = [] + changes = [] + for order, new_author in enumerate(new_rfcauthors): + matching_author = _find_matching_author(new_author, original_authors) + if matching_author is not None: + # Update existing matching author using new_author data + authors_to_commit.append(matching_author) + original_authors.remove(matching_author) # avoid reuse + # Describe changes to this author + author_changes = [] + # Update fields other than order + for field in ["titlepage_name", "is_editor", "affiliation", "country"]: + author_changes.append( + _change_field_and_describe( + matching_author, + field, + getattr(new_author, field), + # List titlepage_name as "name" in logs + "name" if field == "titlepage_name" else field, + ) + ) + # Update order + author_changes.append( + _change_field_and_describe(matching_author, "order", order + 1) + ) + matching_author.save() + author_change_summary = ", ".join( + [ch for ch in author_changes if ch is not None] + ) + if len(author_change_summary) > 0: + changes.append( + 'Changed author "{name}": {summary}'.format( + name=matching_author.titlepage_name, + summary=author_change_summary, + ) + ) + else: + # No author matched, so update the new_author and use that + new_author.document = rfc + new_author.order = order + 1 + new_author.save() + if new_author.person_id is not None: + person_desc = f"Person {new_author.person_id}" + else: + person_desc = "no Person linked" + changes.append( + f'Added "{new_author.titlepage_name}" ({person_desc}) as author' + ) + # Any authors left in original_authors are no longer in the list, so remove them + for removed_author in original_authors: + # Skip actual removal of old authors if we are converting from the + # DocumentAuthor models - the original_authors were just stand-ins anyway. + if not converting_from_docauthors: + removed_author.delete() + if removed_author.person_id is not None: + person_desc = f"Person {removed_author.person_id}" + else: + person_desc = "no Person linked" + changes.append( + f'Removed "{removed_author.titlepage_name}" ({person_desc}) as author' + ) + # Create DocEvents, but leave it up to caller to save + if by is None: + by = Person.objects.get(name="(System)") + return [ + EditedRfcAuthorsDocEvent( + type="edited_authors", + by=by, + doc=rfc, + desc=change, + ) + for change in changes + ] + + def update_reminder(doc, reminder_type_slug, event, due_date): reminder_type = DocReminderTypeName.objects.get(slug=reminder_type_slug) @@ -687,6 +829,22 @@ def nice_consensus(consensus): } return mapping[consensus] +def last_ballot_doc_revision(doc, person): + """ Return the document revision for the most recent ballot position + by the provided user. """ + ballot = doc.active_ballot() + if ballot is None or person is None: + return None + balloters = get_active_balloters(ballot.ballot_type) + if person not in balloters: + return None + position_queryset = BallotPositionDocEvent.objects.filter(type="changed_ballot_position", balloter=person, ballot=ballot).order_by("-time") + if not position_queryset.exists(): + return None + ballot_time = position_queryset.first().time + doc_rev = NewRevisionDocEvent.objects.filter(doc=doc, time__lte=ballot_time).order_by('-time').first().rev + return doc_rev + def has_same_ballot(doc, date1, date2=None): """ Test if the most recent ballot created before the end of date1 is the same as the most recent ballot created before the @@ -771,7 +929,7 @@ def update_telechat(request, doc, by, new_telechat_date, new_returning_item=None else: e.desc = "Removed from agenda for telechat" elif on_agenda and new_telechat_date != prev_telechat: - e.desc = "Telechat date has been changed to %s from %s" % ( + e.desc = "Telechat date has been changed to %s (Previous date was %s)" % ( new_telechat_date, prev_telechat) else: # we didn't reschedule but flipped returning item bit - let's @@ -799,50 +957,93 @@ def rebuild_reference_relations(doc, filenames): filenames should be a dict mapping file ext (i.e., type) to the full path of each file. """ - if doc.type.slug != 'draft': + if doc.type.slug not in ["draft", "rfc"]: + log.log(f"rebuild_reference_relations called for non draft/rfc doc {doc.name}") return None - # try XML first - if 'xml' in filenames: - refs = XMLDraft(filenames['xml']).get_refs() - elif 'txt' in filenames: - filename = filenames['txt'] - try: - refs = draft.PlaintextDraft.from_file(filename).get_refs() - except IOError as e: - return { 'errors': ["%s :%s" % (e.strerror, filename)] } - else: - return {'errors': ['No Internet-Draft text available for rebuilding reference relations. Need XML or plaintext.']} - doc.relateddocument_set.filter(relationship__slug__in=['refnorm','refinfo','refold','refunk']).delete() + if "xml" not in filenames and "txt" not in filenames: + log.log(f"rebuild_reference_relations error: no file available for {doc.name}") + return { + "errors": [ + "No file available for rebuilding reference relations. Need XML or plaintext." + ] + } + else: + try: + # try XML first + if "xml" in filenames: + refs = XMLDraft(filenames["xml"]).get_refs() + elif "txt" in filenames: + filename = filenames["txt"] + refs = draft.PlaintextDraft.from_file(filename).get_refs() + except (IOError, UnicodeDecodeError) as e: + log.log(f"rebuild_reference_relations error: On {doc.name}: {e}") + return {"errors": [f"{e}: {filename}"]} + + before = set(doc.relateddocument_set.filter( + relationship__slug__in=["refnorm", "refinfo", "refold", "refunk"] + ).values_list("relationship__slug","target__name")) warnings = [] errors = [] unfound = set() - for ( ref, refType ) in refs.items(): - refdoc = Document.objects.filter(name=ref) - if not refdoc and re.match(r"^draft-.*-\d{2}$", ref): - refdoc = Document.objects.filter(name=ref[:-3]) + intended = set() + names = [ref for ref in refs] + names.extend([ref[:-3] for ref in refs if re.match(r"^draft-.*-\d{2}$", ref)]) + queryset = Document.objects.filter(name__in=names) + for ref, refType in refs.items(): + refdoc = queryset.filter(name=ref) + if not refdoc.exists() and re.match(r"^draft-.*-\d{2}$", ref): + refdoc = queryset.filter(name=ref[:-3]) count = refdoc.count() if count == 0: - unfound.add( "%s" % ref ) + unfound.add("%s" % ref) continue elif count > 1: - errors.append("Too many Document objects found for %s"%ref) + log.unreachable("2026-3-16") # This branch is holdover from DocAlias + errors.append("Too many Document objects found for %s" % ref) else: # Don't add references to ourself if doc != refdoc[0]: - RelatedDocument.objects.get_or_create( source=doc, target=refdoc[ 0 ], relationship=DocRelationshipName.objects.get( slug='ref%s' % refType ) ) + intended.add((f"ref{refType}", refdoc[0].name)) + if unfound: - warnings.append('There were %d references with no matching Document'%len(unfound)) + warnings.append( + "There were %d references with no matching Document" % len(unfound) + ) + + if intended != before: + for slug, name in before-intended: + doc.relateddocument_set.filter(target__name=name, relationship_id=slug).delete() + for slug, name in intended-before: + doc.relateddocument_set.create( + target=queryset.get(name=name), + relationship_id=slug + ) + after = set(doc.relateddocument_set.filter( + relationship__slug__in=["refnorm", "refinfo", "refold", "refunk"] + ).values_list("relationship__slug","target__name")) + if after != intended: + errors.append("Attempted changed didn't achieve intended results") + changed_references = True + else: + changed_references = False ret = {} if errors: - ret['errors']=errors + ret["errors"] = errors if warnings: - ret['warnings']=warnings + ret["warnings"] = warnings if unfound: - ret['unfound']=list(unfound) + ret["unfound"] = list(unfound) + + logmsg = f"rebuild_reference_relations for {doc.name}: " + logmsg += "changed references" if changed_references else "references unchanged" + if ret: + logmsg += f" {json.dumps(ret)}" + + log.log(logmsg) return ret @@ -1073,9 +1274,6 @@ def build_file_urls(doc: Union[Document, DocHistory]): label = "plain text" if t == "txt" else t file_urls.append((label, base + doc.name + "." + t)) - if "pdf" not in found_types and "txt" in found_types: - file_urls.append(("pdf", base + "pdfrfc/" + doc.name + ".txt.pdf")) - if "txt" in found_types: file_urls.append(("htmlized", urlreverse('ietf.doc.views_doc.document_html', kwargs=dict(name=doc.name)))) if doc.tags.filter(slug="verified-errata").exists(): @@ -1515,3 +1713,23 @@ def update_or_create_draft_bibxml_file(doc, rev): def ensure_draft_bibxml_path_exists(): (Path(settings.BIBXML_BASE_PATH) / "bibxml-ids").mkdir(exist_ok=True) + + +def replicate_stored_objects_for_document(doc: Document) -> int: + """Sync all StoredObjects associated with doc to the replica blob store + + Returns count of StoredObjects queued for replication (which may or may not + be replicated, depending on whether replication is enabled / the storages are + actually BlobdbStorage instances, etc). + """ + # n.b., StoredObjects have a nullable doc_rev field, but Documents do not. + # Until / unless we straighten that out, treat "" and None equivalently when + # matching rev. + qs_matching_rev = StoredObject.objects.filter(doc_rev=doc.rev) + if doc.rev == "": + qs_matching_rev |= StoredObject.objects.filter(doc_rev__isnull=True) + count = 0 + for stored_object in qs_matching_rev.filter(doc_name=doc.name): + force_replication(kind=stored_object.store, name=stored_object.name) + count += 1 + return count diff --git a/ietf/doc/utils_bofreq.py b/ietf/doc/utils_bofreq.py index aec8f60ad6..d01b039b8e 100644 --- a/ietf/doc/utils_bofreq.py +++ b/ietf/doc/utils_bofreq.py @@ -1,12 +1,149 @@ -# Copyright The IETF Trust 2021 All Rights Reserved +# Copyright The IETF Trust 2021-2026 All Rights Reserved +import datetime +from pathlib import Path -from ietf.doc.models import BofreqEditorDocEvent, BofreqResponsibleDocEvent +from django.conf import settings + +from ietf.doc.models import ( + BofreqEditorDocEvent, + BofreqResponsibleDocEvent, + DocEvent, + DocHistory, + Document, +) from ietf.person.models import Person +from ietf.utils import log + def bofreq_editors(bofreq): e = bofreq.latest_event(BofreqEditorDocEvent) return e.editors.all() if e else Person.objects.none() + def bofreq_responsible(bofreq): e = bofreq.latest_event(BofreqResponsibleDocEvent) - return e.responsible.all() if e else Person.objects.none() \ No newline at end of file + return e.responsible.all() if e else Person.objects.none() + + +def fixup_bofreq_timestamps(): # pragma: nocover + """Fixes bofreq event / document timestamps + + Timestamp errors resulted from the bug fixed by + https://github.com/ietf-tools/datatracker/pull/10333 + + Does not fix up -00 revs because the timestamps on these were not affected by + the bug. Replacing their timestamps creates a confusing event history because the + filesystem timestamp is usually a fraction of a second later than other events + created upon the initial rev creation. This causes the "New revision available" + event to appear _after_ these events in the history. Better to leave them as is. + """ + FIX_DEPLOYMENT_TIME = "2026-02-03T01:16:00+00:00" # 12.58.0 -> production + + def _get_doc_time(doc_name: str, rev: str): + path = Path(settings.BOFREQ_PATH) / f"{doc_name}-{rev}.md" + return datetime.datetime.fromtimestamp(path.stat().st_mtime, datetime.UTC) + + # Find affected DocEvents and DocHistories + new_bofreq_events = ( + DocEvent.objects.filter( + doc__type="bofreq", type="new_revision", time__lt=FIX_DEPLOYMENT_TIME + ) + .exclude(rev="00") # bug did not affect rev 00 events + .order_by("doc__name", "rev") + ) + log.log( + f"fixup_bofreq_timestamps: found {new_bofreq_events.count()} " + f"new_revision events before {FIX_DEPLOYMENT_TIME}" + ) + document_fixups = {} + for e in new_bofreq_events: + name = e.doc.name + rev = e.rev + filesystem_time = _get_doc_time(name, rev) + assert e.time < filesystem_time, ( + f"Rev {rev} event timestamp for {name} unexpectedly later than the " + "filesystem timestamp!" + ) + try: + dochistory = DocHistory.objects.filter( + name=name, time__lt=filesystem_time + ).get(rev=rev) + except DocHistory.MultipleObjectsReturned as err: + raise RuntimeError( + f"Multiple DocHistories for {name} rev {rev} exist earlier than the " + "filesystem timestamp!" + ) from err + except DocHistory.DoesNotExist as err: + if rev == "00": + # Unreachable because we don't adjust -00 revs, but could be needed + # if we did, in theory. In practice it's still not reached, but + # keeping the case for completeness. + dochistory = None + else: + raise RuntimeError( + f"No DocHistory for {name} rev {rev} exists earlier than the " + f"filesystem timestamp!" + ) from err + + if name not in document_fixups: + document_fixups[name] = [] + document_fixups[name].append( + { + "event": e, + "dochistory": dochistory, + "filesystem_time": filesystem_time, + } + ) + + # Now do the actual fixup + system_person = Person.objects.get(name="(System)") + for doc_name, fixups in document_fixups.items(): + bofreq = Document.objects.get(type="bofreq", name=doc_name) + log_msg_parts = [] + adjusted_revs = [] + for fixup in fixups: + event_to_fix = fixup["event"] + dh_to_fix = fixup["dochistory"] + new_time = fixup["filesystem_time"] + adjusted_revs.append(event_to_fix.rev) + + # Fix up the event + event_to_fix.time = new_time + event_to_fix.save() + log_msg_parts.append(f"rev {event_to_fix.rev} DocEvent") + + # Fix up the DocHistory + if dh_to_fix is not None: + dh_to_fix.time = new_time + dh_to_fix.save() + log_msg_parts.append(f"rev {dh_to_fix.rev} DocHistory") + + if event_to_fix.rev == bofreq.rev and bofreq.time < new_time: + # Update the Document without calling save(). Only update if + # the time has not changed so we don't inadvertently overwrite + # a concurrent update. + Document.objects.filter(pk=bofreq.pk, time=bofreq.time).update( + time=new_time + ) + bofreq.refresh_from_db() + if bofreq.rev == event_to_fix.rev: + log_msg_parts.append(f"rev {bofreq.rev} Document") + else: + log.log( + "fixup_bofreq_timestamps: WARNING: bofreq Document rev " + f"changed for {bofreq.name}" + ) + log.log(f"fixup_bofreq_timestamps: {bofreq.name}: " + ", ".join(log_msg_parts)) + + # Fix up the Document, if necessary, and add a record of the adjustment + DocEvent.objects.create( + type="added_comment", + by=system_person, + doc=bofreq, + rev=bofreq.rev, + desc=( + "Corrected inaccurate document and new revision event timestamps for " + + ("version " if len(adjusted_revs) == 1 else "versions ") + + ", ".join(adjusted_revs) + ), + ) diff --git a/ietf/doc/utils_errata.py b/ietf/doc/utils_errata.py new file mode 100644 index 0000000000..539262151f --- /dev/null +++ b/ietf/doc/utils_errata.py @@ -0,0 +1,35 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +import requests + +from django.conf import settings + +from ietf.utils.log import log + + +def signal_update_rfc_metadata(rfc_number_list=()): + key = getattr(settings, "ERRATA_METADATA_NOTIFICATION_API_KEY", None) + if key is not None: + headers = {"X-Api-Key": settings.ERRATA_METADATA_NOTIFICATION_API_KEY} + post_dict = { + "rfc_number_list": list(rfc_number_list), + } + try: + response = requests.post( + settings.ERRATA_METADATA_NOTIFICATION_URL, + headers=headers, + json=post_dict, + timeout=settings.DEFAULT_REQUESTS_TIMEOUT, + ) + except requests.Timeout as e: + log( + f"POST request timed out for {settings.ERRATA_METADATA_NOTIFICATION_URL} ]: {e}" + ) + # raise RuntimeError(f'POST request timed out for {settings.ERRATA_METADATA_NOTIFICATION_URL}') from e + return + if response.status_code != 200: + log( + f"POST request failed for {settings.ERRATA_METADATA_NOTIFICATION_URL} ]: {response.status_code} {response.text}" + ) + else: + log("No API key configured for errata metadata notification, skipping") diff --git a/ietf/doc/utils_r2.py b/ietf/doc/utils_r2.py new file mode 100644 index 0000000000..53fb978303 --- /dev/null +++ b/ietf/doc/utils_r2.py @@ -0,0 +1,17 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.core.files.storage import storages + +from ietf.doc.models import StoredObject + + +def rfcs_are_in_r2(rfc_number_list=()): + r2_rfc_bucket = storages["r2-rfc"] + for rfc_number in rfc_number_list: + stored_objects = StoredObject.objects.filter( + store="rfc", doc_name=f"rfc{rfc_number}" + ) + for stored_object in stored_objects: + if not r2_rfc_bucket.exists(stored_object.name): + return False + return True diff --git a/ietf/doc/utils_red.py b/ietf/doc/utils_red.py new file mode 100644 index 0000000000..5c5879d688 --- /dev/null +++ b/ietf/doc/utils_red.py @@ -0,0 +1,31 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +import requests + +from django.conf import settings + +from ietf.utils.log import log + + +def trigger_red_precomputer(rfc_number_list=()): + url = getattr(settings, "TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL", None) + if url is not None: + payload = { + "rfcs": ",".join([str(n) for n in rfc_number_list]), + } + try: + log(f"Triggering red precompute multiple for RFCs {rfc_number_list}") + response = requests.post( + url=url, + json=payload, + timeout=settings.DEFAULT_REQUESTS_TIMEOUT, + ) + except requests.Timeout as e: + log(f"POST request timed out for {url} : {e}") + return + if response.status_code // 100 != 2: # 2xx status codes are ok + log( + f"POST request failed for {url} : status_code={response.status_code}" + ) + else: + log("No URL configured for triggering red precompute multiple, skipping") diff --git a/ietf/doc/utils_search.py b/ietf/doc/utils_search.py index 59b64ad307..a5f461f9bb 100644 --- a/ietf/doc/utils_search.py +++ b/ietf/doc/utils_search.py @@ -108,7 +108,10 @@ def fill_in_document_table_attributes(docs, have_telechat_date=False): d.search_heading = "Withdrawn Internet-Draft" d.expirable = False else: - d.search_heading = "%s Internet-Draft" % d.get_state() + if d.type_id == "draft" and d.stream_id == 'ietf' and d.get_state_slug('draft-iesg') != 'idexists': # values can be: ad-eval idexists approved rfcqueue dead iesg-eva + d.search_heading = "%s with the IESG Internet-Draft" % d.get_state() + else: + d.search_heading = "%s Internet-Draft" % d.get_state() if state_slug == "active": d.expirable = d.pk in expirable_pks else: @@ -221,6 +224,10 @@ def num(i): if d.type_id == "draft": res.append(num(["Active", "Expired", "Replaced", "Withdrawn", "RFC"].index(d.search_heading.split()[0]))) + if "with the IESG" in d.search_heading: + res.append("1") + else: + res.append("0") else: res.append(d.type_id); res.append("-"); @@ -299,10 +306,10 @@ def num(i): "ad-eval", "lc-req", "lc", + "goaheadw", "writeupw", # "defer", # probably not a useful state to show, since it's rare "iesg-eva", - "goaheadw", "approved", "ann", ], diff --git a/ietf/doc/views_ballot.py b/ietf/doc/views_ballot.py index 9e2a417933..29aadfdb9b 100644 --- a/ietf/doc/views_ballot.py +++ b/ietf/doc/views_ballot.py @@ -4,18 +4,18 @@ # Directors and Secretariat -import datetime, json +import datetime +import json from django import forms from django.conf import settings -from django.http import HttpResponse, HttpResponseRedirect, Http404, HttpResponseBadRequest +from django.http import HttpResponse, HttpResponseNotAllowed, HttpResponseRedirect, Http404, HttpResponseBadRequest from django.shortcuts import render, get_object_or_404, redirect from django.template.defaultfilters import striptags from django.template.loader import render_to_string from django.urls import reverse as urlreverse from django.views.decorators.csrf import csrf_exempt from django.utils.html import escape -from urllib.parse import urlencode as urllib_urlencode import debug # pyflakes:ignore @@ -34,14 +34,15 @@ from ietf.doc.templatetags.ietf_filters import can_ballot from ietf.iesg.models import TelechatDate from ietf.ietfauth.utils import has_role, role_required, is_authorized_in_doc_stream +from ietf.mailtrigger.models import Recipient from ietf.mailtrigger.utils import gather_address_lists from ietf.mailtrigger.forms import CcSelectForm from ietf.message.utils import infer_message from ietf.name.models import BallotPositionName, DocTypeName from ietf.person.models import Person -from ietf.utils.fields import ModelMultipleChoiceField +from ietf.utils.fields import ModelMultipleChoiceField, MultiEmailField from ietf.utils.http import validate_return_to_path -from ietf.utils.mail import send_mail_text, send_mail_preformatted +from ietf.utils.mail import decode_header_value, send_mail_text, send_mail_preformatted from ietf.utils.decorators import require_api_key from ietf.utils.response import permission_denied from ietf.utils.timezone import date_today, datetime_from_date, DEADLINE_TZINFO @@ -179,6 +180,9 @@ def save_position(form, doc, ballot, balloter, login=None, send_email=False): return pos +class AdditionalCCForm(forms.Form): + additional_cc = MultiEmailField(required=False) + @role_required("Area Director", "Secretariat", "IRSG Member", "RSAB Member") def edit_position(request, name, ballot_id): """Vote and edit discuss and comment on document""" @@ -199,50 +203,67 @@ def edit_position(request, name, ballot_id): raise Http404 balloter = get_object_or_404(Person, pk=balloter_id) + if doc.stream_id == 'irtf': + mailtrigger_slug='irsg_ballot_saved' + elif doc.stream_id == 'editorial': + mailtrigger_slug='rsab_ballot_saved' + else: + mailtrigger_slug='iesg_ballot_saved' + if request.method == 'POST': old_pos = None if not has_role(request.user, "Secretariat") and not can_ballot(request.user, doc): # prevent pre-ADs from taking a position permission_denied(request, "Must be an active member (not a pre-AD for example) of the balloting body to take a position") + if request.POST.get("Defer") and doc.stream.slug != "irtf": + return redirect('ietf.doc.views_ballot.defer_ballot', name=doc) + elif request.POST.get("Undefer") and doc.stream.slug != "irtf": + return redirect('ietf.doc.views_ballot.undefer_ballot', name=doc) + form = EditPositionForm(request.POST, ballot_type=ballot.ballot_type) - if form.is_valid(): + cc_select_form = CcSelectForm(data=request.POST,mailtrigger_slug=mailtrigger_slug,mailtrigger_context={'doc':doc}) + additional_cc_form = AdditionalCCForm(request.POST) + if form.is_valid() and cc_select_form.is_valid() and additional_cc_form.is_valid(): send_mail = True if request.POST.get("send_mail") else False - save_position(form, doc, ballot, balloter, login, send_mail) - + pos = save_position(form, doc, ballot, balloter, login, send_mail) if send_mail: - query = {} - if request.GET.get('balloter'): - query['balloter'] = request.GET.get('balloter') - if request.GET.get('ballot_edit_return_point'): - query['ballot_edit_return_point'] = request.GET.get('ballot_edit_return_point') - qstr = "" - if len(query) > 0: - qstr = "?" + urllib_urlencode(query, safe='/') - return HttpResponseRedirect(urlreverse('ietf.doc.views_ballot.send_ballot_comment', kwargs=dict(name=doc.name, ballot_id=ballot_id)) + qstr) - elif request.POST.get("Defer") and doc.stream.slug != "irtf": - return redirect('ietf.doc.views_ballot.defer_ballot', name=doc) - elif request.POST.get("Undefer") and doc.stream.slug != "irtf": - return redirect('ietf.doc.views_ballot.undefer_ballot', name=doc) - else: - return HttpResponseRedirect(return_to_url) + addrs, frm, subject, body = build_position_email(balloter, doc, pos) + if doc.stream_id == 'irtf': + mailtrigger_slug='irsg_ballot_saved' + elif doc.stream_id == 'editorial': + mailtrigger_slug='rsab_ballot_saved' + else: + mailtrigger_slug='iesg_ballot_saved' + cc = [] + cc.extend(cc_select_form.get_selected_addresses()) + extra_cc = additional_cc_form.cleaned_data["additional_cc"] + if extra_cc: + cc.extend(extra_cc) + cc_set = set(cc) + cc_set.discard("") + cc = sorted(list(cc_set)) + send_mail_text(request, addrs.to, frm, subject, body, cc=", ".join(cc)) + return redirect(return_to_url) else: initial = {} old_pos = doc.latest_event(BallotPositionDocEvent, type="changed_ballot_position", balloter=balloter, ballot=ballot) if old_pos: initial['position'] = old_pos.pos.slug initial['discuss'] = old_pos.discuss - initial['comment'] = old_pos.comment - + initial['comment'] = old_pos.comment form = EditPositionForm(initial=initial, ballot_type=ballot.ballot_type) + cc_select_form = CcSelectForm(mailtrigger_slug=mailtrigger_slug,mailtrigger_context={'doc':doc}) + additional_cc_form = AdditionalCCForm() blocking_positions = dict((p.pk, p.name) for p in form.fields["position"].queryset.all() if p.blocking) - ballot_deferred = doc.active_defer_event() return render(request, 'doc/ballot/edit_position.html', dict(doc=doc, form=form, + cc_select_form=cc_select_form, + additional_cc_form=additional_cc_form, balloter=balloter, return_to_url=return_to_url, old_pos=old_pos, @@ -257,7 +278,11 @@ def edit_position(request, name, ballot_id): @csrf_exempt def api_set_position(request): def err(code, text): - return HttpResponse(text, status=code, content_type='text/plain') + return HttpResponse( + text, + status=code, + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + ) if request.method == 'POST': ad = request.user.person name = request.POST.get('doc') @@ -290,24 +315,105 @@ def err(code, text): addrs, frm, subject, body = build_position_email(ad, doc, pos) send_mail_text(request, addrs.to, frm, subject, body, cc=addrs.cc) - return HttpResponse("Done", status=200, content_type='text/plain') + return HttpResponse( + "Done", + status=200, + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + ) -def build_position_email(balloter, doc, pos): +@role_required("Area Director", "Secretariat") +@csrf_exempt +def ajax_build_position_email(request): + if request.method != "POST": + return HttpResponseNotAllowed(["POST"]) + errors = list() + try: + json_body = json.loads(request.body) + except json.decoder.JSONDecodeError: + errors.append("Post body is not valid json") + if len(errors) == 0: + post_data = json_body.get("post_data") + if post_data is None: + errors.append("post_data not provided") + else: + for key in [ + "discuss", + "comment", + "position", + "balloter", + "docname", + "cc_choices", + "additional_cc", + ]: + if key not in post_data: + errors.append(f"{key} not found in post_data") + if len(errors) == 0: + person = Person.objects.filter(pk=post_data.get("balloter")).first() + if person is None: + errors.append("No person found matching balloter") + doc = Document.objects.filter(name=post_data.get("docname")).first() + if doc is None: + errors.append("No document found matching docname") + if len(errors) > 0: + response = { + "success": False, + "errors": errors, + } + else: + wanted = dict() # consider named tuple instead + wanted["discuss"] = post_data.get("discuss") + wanted["comment"] = post_data.get("comment") + wanted["position_name"] = post_data.get("position") + wanted["balloter"] = person + wanted["doc"] = doc + addrs, frm, subject, body = build_position_email_from_dict(wanted) + + recipient_slugs = post_data.get("cc_choices") + # Consider refactoring gather_address_lists so this isn't duplicated from there + cc_addrs = set() + for r in Recipient.objects.filter(slug__in=recipient_slugs): + cc_addrs.update(r.gather(doc=doc)) + additional_cc = post_data.get("additional_cc") + for addr in additional_cc.split(","): + cc_addrs.add(addr.strip()) + cc_addrs.discard("") + cc_addrs = sorted(list(cc_addrs)) + + response_text = "\n".join( + [ + f"From: {decode_header_value(frm)}", + f"To: {', '.join([decode_header_value(addr) for addr in addrs.to])}", + f"Cc: {', '.join([decode_header_value(addr) for addr in cc_addrs])}", + f"Subject: {subject}", + "", + body, + ] + ) + + response = { + "success": True, + "text": response_text, + } + return HttpResponse(json.dumps(response), content_type="application/json") + +def build_position_email_from_dict(pos_dict): + doc = pos_dict["doc"] subj = [] d = "" blocking_name = "DISCUSS" - if pos.pos.blocking and pos.discuss: - d = pos.discuss - blocking_name = pos.pos.name.upper() + pos_name = BallotPositionName.objects.filter(slug=pos_dict["position_name"]).first() + if pos_name.blocking and pos_dict.get("discuss"): + d = pos_dict.get("discuss") + blocking_name = pos_name.name.upper() subj.append(blocking_name) c = "" - if pos.comment: - c = pos.comment + if pos_dict.get("comment"): + c = pos_dict.get("comment") subj.append("COMMENT") - + balloter = pos_dict.get("balloter") balloter_name_genitive = balloter.plain_name() + "'" if balloter.plain_name().endswith('s') else balloter.plain_name() + "'s" - subject = "%s %s on %s" % (balloter_name_genitive, pos.pos.name if pos.pos else "No Position", doc.name + "-" + doc.rev) + subject = "%s %s on %s" % (balloter_name_genitive, pos_name.name if pos_name else "No Position", doc.name + "-" + doc.rev) if subj: subject += ": (with %s)" % " and ".join(subj) @@ -316,7 +422,7 @@ def build_position_email(balloter, doc, pos): comment=c, balloter=balloter.plain_name(), doc=doc, - pos=pos.pos, + pos=pos_name, blocking_name=blocking_name, settings=settings)) frm = balloter.role_email("ad").formatted_email() @@ -330,79 +436,16 @@ def build_position_email(balloter, doc, pos): return addrs, frm, subject, body -@role_required('Area Director','Secretariat','IRSG Member', 'RSAB Member') -def send_ballot_comment(request, name, ballot_id): - """Email document ballot position discuss/comment for Area Director.""" - doc = get_object_or_404(Document, name=name) - ballot = get_object_or_404(BallotDocEvent, type="created_ballot", pk=ballot_id, doc=doc) - - if not has_role(request.user, 'Secretariat'): - if any([ - doc.stream_id == 'ietf' and not has_role(request.user, 'Area Director'), - doc.stream_id == 'irtf' and not has_role(request.user, 'IRSG Member'), - doc.stream_id == 'editorial' and not has_role(request.user, 'RSAB Member'), - ]): - raise Http404 - - balloter = request.user.person - try: - return_to_url = parse_ballot_edit_return_point(request.GET.get('ballot_edit_return_point'), doc.name, ballot_id) - except ValueError: - return HttpResponseBadRequest('ballot_edit_return_point is invalid') - - if 'HTTP_REFERER' in request.META: - back_url = request.META['HTTP_REFERER'] - else: - back_url = urlreverse("ietf.doc.views_doc.document_ballot", kwargs=dict(name=doc.name, ballot_id=ballot_id)) - - # if we're in the Secretariat, we can select a balloter (such as an AD) to act as stand-in for - if has_role(request.user, "Secretariat"): - balloter_id = request.GET.get('balloter') - if not balloter_id: - raise Http404 - balloter = get_object_or_404(Person, pk=balloter_id) - - pos = doc.latest_event(BallotPositionDocEvent, type="changed_ballot_position", balloter=balloter, ballot=ballot) - if not pos: - raise Http404 - - addrs, frm, subject, body = build_position_email(balloter, doc, pos) - - if doc.stream_id == 'irtf': - mailtrigger_slug='irsg_ballot_saved' - elif doc.stream_id == 'editorial': - mailtrigger_slug='rsab_ballot_saved' - else: - mailtrigger_slug='iesg_ballot_saved' - - if request.method == 'POST': - cc = [] - cc_select_form = CcSelectForm(data=request.POST,mailtrigger_slug=mailtrigger_slug,mailtrigger_context={'doc':doc}) - if cc_select_form.is_valid(): - cc.extend(cc_select_form.get_selected_addresses()) - extra_cc = [x.strip() for x in request.POST.get("extra_cc","").split(',') if x.strip()] - if extra_cc: - cc.extend(extra_cc) - - send_mail_text(request, addrs.to, frm, subject, body, cc=", ".join(cc)) - - return HttpResponseRedirect(return_to_url) +def build_position_email(balloter, doc, pos): - else: - - cc_select_form = CcSelectForm(mailtrigger_slug=mailtrigger_slug,mailtrigger_context={'doc':doc}) - - return render(request, 'doc/ballot/send_ballot_comment.html', - dict(doc=doc, - subject=subject, - body=body, - frm=frm, - to=addrs.as_strings().to, - balloter=balloter, - back_url=back_url, - cc_select_form = cc_select_form, - )) + pos_dict=dict() + pos_dict["doc"]=doc + pos_dict["position_name"]=pos.pos.slug + pos_dict["discuss"]=pos.discuss + pos_dict["comment"]=pos.comment + pos_dict["balloter"]=balloter + return build_position_email_from_dict(pos_dict) @role_required('Area Director','Secretariat') def clear_ballot(request, name, ballot_type_slug): @@ -896,16 +939,6 @@ def approve_ballot(request, name): if ballot_writeup_event.pk == None: ballot_writeup_event.save() - if new_state.slug == "ann" and new_state.slug != prev_state.slug: - # start by notifying the RFC Editor - import ietf.sync.rfceditor - response, error = ietf.sync.rfceditor.post_approved_draft(settings.RFC_EDITOR_SYNC_NOTIFICATION_URL, doc.name) - if error: - return render(request, 'doc/draft/rfceditor_post_approved_draft_failed.html', - dict(name=doc.name, - response=response, - error=error)) - doc.set_state(new_state) doc.tags.remove(*prev_tags) diff --git a/ietf/doc/views_bofreq.py b/ietf/doc/views_bofreq.py index 71cbe30491..94e3960dfa 100644 --- a/ietf/doc/views_bofreq.py +++ b/ietf/doc/views_bofreq.py @@ -91,7 +91,6 @@ def submit(request, name): by=request.user.person, rev=bofreq.rev, desc='New revision available', - time=bofreq.time, ) bofreq.save_with_history([e]) bofreq_submission = form.cleaned_data['bofreq_submission'] diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index 591a72d907..af056f6a96 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2009-2024, All Rights Reserved +# Copyright The IETF Trust 2009-2026, All Rights Reserved # -*- coding: utf-8 -*- # # Parts Copyright (C) 2009-2010 Nokia Corporation and/or its subsidiary(-ies). @@ -43,9 +43,10 @@ from celery.result import AsyncResult from django.core.cache import caches +from django.core.files.base import ContentFile from django.core.exceptions import PermissionDenied from django.db.models import Max -from django.http import HttpResponse, Http404, HttpResponseBadRequest, JsonResponse +from django.http import FileResponse, HttpResponse, Http404, HttpResponseBadRequest, HttpResponseForbidden, JsonResponse from django.shortcuts import render, get_object_or_404, redirect from django.template.loader import render_to_string from django.urls import reverse as urlreverse @@ -57,7 +58,7 @@ import debug # pyflakes:ignore from ietf.doc.models import ( Document, DocHistory, DocEvent, BallotDocEvent, BallotType, - ConsensusDocEvent, NewRevisionDocEvent, TelechatDocEvent, WriteupDocEvent, IanaExpertDocEvent, + ConsensusDocEvent, NewRevisionDocEvent, StoredObject, TelechatDocEvent, WriteupDocEvent, IanaExpertDocEvent, IESG_BALLOT_ACTIVE_STATES, STATUSCHANGE_RELATIONS, DocumentActionHolder, DocumentAuthor, RelatedDocument, RelatedDocHistory) from ietf.doc.tasks import investigate_fragment_task @@ -79,19 +80,21 @@ from ietf.doc.views_ballot import parse_ballot_edit_return_point from ietf.doc.forms import InvestigateForm, TelechatForm, NotifyForm, ActionHoldersForm, DocAuthorForm, DocAuthorChangeBasisForm from ietf.doc.mails import email_comment, email_remind_action_holders +from ietf.doc.utils import last_ballot_doc_revision from ietf.mailtrigger.utils import gather_relevant_expansions from ietf.meeting.models import Session, SessionPresentation from ietf.meeting.utils import group_sessions, get_upcoming_manageable_sessions, sort_sessions, add_event_info_to_session_qs from ietf.review.models import ReviewAssignment from ietf.review.utils import can_request_review_of_doc, review_assignments_to_list_for_docs, review_requests_to_list_for_docs from ietf.review.utils import no_review_from_teams_on_doc +from ietf.doc.storage_utils import retrieve_bytes from ietf.utils import markup_txt, log, markdown from ietf.utils.draft import get_status_from_draft_text from ietf.utils.meetecho import MeetechoAPIError, SlidesManager from ietf.utils.response import permission_denied from ietf.utils.text import maybe_split from ietf.utils.timezone import date_today - +from ietf.utils.unicodenormalize import normalize_for_sorting def render_document_top(request, doc, tab, name): tabs = [] @@ -255,7 +258,7 @@ def document_main(request, name, rev=None, document_html=False): interesting_relations_that, interesting_relations_that_doc = interesting_doc_relations(doc) can_edit = has_role(request.user, ("Area Director", "Secretariat")) - can_edit_authors = has_role(request.user, ("Secretariat")) + can_edit_authors = has_role(request.user, ("Secretariat")) and not doc.rfcauthor_set.exists() stream_slugs = StreamName.objects.values_list("slug", flat=True) # For some reason, AnonymousUser has __iter__, but is not iterable, @@ -514,13 +517,17 @@ def document_main(request, name, rev=None, document_html=False): # remaining actions actions = [] - if can_adopt_draft(request.user, doc) and not doc.get_state_slug() in ["rfc"] and not snapshot: + if can_adopt_draft(request.user, doc) and doc.get_state_slug() not in ["rfc"] and not snapshot: + target = urlreverse("ietf.doc.views_draft.adopt_draft", kwargs=dict(name=doc.name)) if doc.group and doc.group.acronym != 'none': # individual submission # already adopted in one group button_text = "Switch adoption" else: button_text = "Manage adoption" - actions.append((button_text, urlreverse('ietf.doc.views_draft.adopt_draft', kwargs=dict(name=doc.name)))) + # can_adopt_draft currently returns False for Area Directors + if has_role(request.user, ["Secretariat", "WG Chair"]): + target = urlreverse("ietf.doc.views_draft.ask_about_ietf_adoption_call", kwargs=dict(name=doc.name)) + actions.append((button_text, target)) if can_unadopt_draft(request.user, doc) and not doc.get_state_slug() in ["rfc"] and not snapshot: if doc.get_state_slug('draft-iesg') == 'idexists': @@ -1225,6 +1232,10 @@ def document_history(request, name): request.user, ("Area Director", "Secretariat", "IRTF Chair") ) + # if the current user has balloted on this document, give them a revision hint + ballot_doc_rev = None + if request.user.is_authenticated: + ballot_doc_rev = last_ballot_doc_revision(doc, request.user.person) return render( request, @@ -1235,6 +1246,7 @@ def document_history(request, name): "diff_revisions": diff_revisions, "events": events, "can_add_comment": can_add_comment, + "ballot_doc_rev": ballot_doc_rev, }, ) @@ -1275,9 +1287,7 @@ def document_bibtex(request, name, rev=None): break elif doc.type_id == "rfc": - # This needs to be replaced with a lookup, as the mapping may change - # over time. - doi = f"10.17487/RFC{doc.rfc_number:04d}" + doi = doc.doi if doc.is_dochistory(): latest_event = doc.latest_event(type='new_revision', rev=rev) @@ -1500,7 +1510,7 @@ def document_ballot_content(request, doc, ballot_id, editable=True): position_groups = [] for n in BallotPositionName.objects.filter(slug__in=[p.pos_id for p in positions]).order_by('order'): g = (n, [p for p in positions if p.pos_id == n.slug]) - g[1].sort(key=lambda p: (p.is_old_pos, p.balloter.plain_name())) + g[1].sort(key=lambda p: (p.is_old_pos, normalize_for_sorting(p.balloter.plain_name()))) if n.blocking: position_groups.insert(0, g) else: @@ -1643,11 +1653,18 @@ def extract_name(s): data["state"] = extract_name(doc.get_state()) data["intended_std_level"] = extract_name(doc.intended_std_level) data["std_level"] = extract_name(doc.std_level) + author_qs = ( + doc.rfcauthor_set + if doc.type_id == "rfc" and doc.rfcauthor_set.exists() + else doc.documentauthor_set + ).select_related("person").prefetch_related("person__email_set").order_by("order") data["authors"] = [ - dict(name=author.person.name, - email=author.email.address if author.email else None, - affiliation=author.affiliation) - for author in doc.documentauthor_set.all().select_related("person", "email").order_by("order") + { + "name": author.titlepage_name if hasattr(author, "titlepage_name") else author.person.name, + "email": author.email.address if author.email else None, + "affiliation": author.affiliation, + } + for author in author_qs ] data["shepherd"] = doc.shepherd.formatted_email() if doc.shepherd else None data["ad"] = doc.ad.role_email("ad").formatted_email() if doc.ad else None @@ -1687,7 +1704,7 @@ def add_comment(request, name): group__acronym=doc.group.acronym, person__user=request.user))) else: - can_add_comment = has_role(request.user, ("Area Director", "Secretariat", "IRTF Chair")) + can_add_comment = has_role(request.user, ("Area Director", "Secretariat", "IRTF Chair", "RFC Editor")) if not can_add_comment: # The user is a chair or secretary, but not for this WG or RG permission_denied(request, "You need to be a chair or secretary of this group to add a comment.") @@ -1825,12 +1842,15 @@ def add_fields(self, form, index): if fh in form.fields: form.fields[fh].widget = forms.HiddenInput() + doc = get_object_or_404(Document, name=name) + if doc.rfcauthor_set.exists(): + return HttpResponseForbidden("Contact the RFC Editor to change RFC Author information") + AuthorFormSet = forms.formset_factory(DocAuthorForm, formset=_AuthorsBaseFormSet, can_delete=True, can_order=True, extra=0) - doc = get_object_or_404(Document, name=name) if request.method == 'POST': change_basis_form = DocAuthorChangeBasisForm(request.POST) @@ -1931,9 +1951,9 @@ def edit_action_holders(request, name): role_ids = dict() # maps role slug to list of Person IDs (assumed numeric in the JavaScript) extra_prefetch = [] # list of Person objects to prefetch for select2 field - if len(doc.authors()) > 0: + authors = doc.author_persons() + if len(authors) > 0: doc_role_labels.append(dict(slug='authors', label='Authors')) - authors = doc.authors() role_ids['authors'] = [p.pk for p in authors] extra_prefetch += authors @@ -2341,3 +2361,29 @@ def investigate(request): "results": results, }, ) + +def rfcxml_notprepped(request, number): + number = int(number) + if number < settings.FIRST_V3_RFC: + raise Http404 + rfc = Document.objects.filter(type="rfc", rfc_number=number).first() + if rfc is None: + raise Http404 + name = f"notprepped/rfc{number}.notprepped.xml" + if not StoredObject.objects.filter(name=name).exists(): + raise Http404 + try: + bytes = retrieve_bytes("rfc", name) + except FileNotFoundError: + raise Http404 + return FileResponse(ContentFile(bytes, name=f"rfc{number}.notprepped.xml"), as_attachment=True) + + +def rfcxml_notprepped_wrapper(request, number): + number = int(number) + if number < settings.FIRST_V3_RFC: + raise Http404 + rfc = Document.objects.filter(type="rfc", rfc_number=number).first() + if rfc is None: + raise Http404 + return render(request, "doc/notprepped_wrapper.html", context={"rfc": rfc}) diff --git a/ietf/doc/views_draft.py b/ietf/doc/views_draft.py index c80537afb3..0f5ea49f5d 100644 --- a/ietf/doc/views_draft.py +++ b/ietf/doc/views_draft.py @@ -33,6 +33,7 @@ email_iesg_processing_document, email_ad_approved_doc, email_iana_expert_review_state_changed ) from ietf.doc.storage_utils import retrieve_bytes, store_bytes +from ietf.doc.templatetags.ietf_filters import is_doc_ietf_adoptable from ietf.doc.utils import ( add_state_change_event, can_adopt_draft, can_unadopt_draft, get_tags_for_stream_id, nice_consensus, update_action_holders, update_reminder, update_telechat, make_notify_changed_event, get_initial_notify, @@ -50,12 +51,12 @@ from ietf.name.models import IntendedStdLevelName, DocTagName, StreamName from ietf.person.fields import SearchableEmailField from ietf.person.models import Person, Email -from ietf.utils.mail import send_mail, send_mail_message, on_behalf_of +from ietf.utils.mail import send_mail, send_mail_message, on_behalf_of, send_mail_text from ietf.utils.textupload import get_cleaned_text_file_content from ietf.utils import log -from ietf.utils.fields import ModelMultipleChoiceField +from ietf.utils.fields import DatepickerDateField, ModelMultipleChoiceField, MultiEmailField from ietf.utils.response import permission_denied -from ietf.utils.timezone import datetime_today, DEADLINE_TZINFO +from ietf.utils.timezone import date_today, datetime_from_date, datetime_today, DEADLINE_TZINFO class ChangeStateForm(forms.Form): @@ -1275,15 +1276,6 @@ class PublicationForm(forms.Form): if form.is_valid(): events = [] - # start by notifying the RFC Editor - import ietf.sync.rfceditor - response, error = ietf.sync.rfceditor.post_approved_draft(settings.RFC_EDITOR_SYNC_NOTIFICATION_URL, doc.name) - if error: - return render(request, 'doc/draft/rfceditor_post_approved_draft_failed.html', - dict(name=doc.name, - response=response, - error=error)) - m.subject = form.cleaned_data["subject"] m.body = form.cleaned_data["body"] m.save() @@ -1563,11 +1555,13 @@ def adopt_draft(request, name): events.append(e) due_date = None - if form.cleaned_data["weeks"] != None: + if form.cleaned_data["weeks"] is not None: due_date = datetime_today(DEADLINE_TZINFO) + datetime.timedelta(weeks=form.cleaned_data["weeks"]) update_reminder(doc, "stream-s", e, due_date) + # The following call name is very misleading - the view allows + # setting states that are _not_ the adopted state. email_adopted(request, doc, prev_state, new_state, by, comment) # comment @@ -1681,11 +1675,14 @@ def __init__(self, *args, **kwargs): f.queryset = f.queryset.exclude(pk__in=unused_states) f.label = state_type.label if self.stream.slug == 'ietf': + help_text_items = [] if self.can_set_sub_pub: - f.help_text = "Only select 'Submitted to IESG for Publication' to correct errors. Use the document's main page to request publication." + help_text_items.append("Only select 'Submitted to IESG for Publication' to correct errors. This is not how to submit a document to the IESG.") else: f.queryset = f.queryset.exclude(slug='sub-pub') - f.help_text = "You may not set the 'Submitted to IESG for Publication' using this form - Use the document's main page to request publication." + help_text_items.append("You may not set the 'Submitted to IESG for Publication' using this form - Use the button above or the document's main page to request publication.") + help_text_items.append("Only use this form in unusual circumstances when issuing call for adoption or working group last call.") + f.help_text = " ".join(help_text_items) f = self.fields['tags'] f.queryset = f.queryset.filter(slug__in=get_tags_for_stream_id(doc.stream_id)) @@ -1696,7 +1693,7 @@ def __init__(self, *args, **kwargs): def clean_new_state(self): new_state = self.cleaned_data.get('new_state') if new_state.slug=='sub-pub' and not self.can_set_sub_pub: - raise forms.ValidationError('You may not set the %s state using this form. Use the "Submit to IESG for publication" button on the document\'s main page instead. If that button does not appear, the document may already have IESG state. Ask your Area Director or the Secretariat for help.'%new_state.name) + raise forms.ValidationError('You may not set the %s state using this form. Use the "Submit to IESG for Publication" button on the document\'s main page instead. If that button does not appear, the document may already have IESG state. Ask your Area Director or the Secretariat for help.'%new_state.name) return new_state @@ -1722,6 +1719,19 @@ def next_states_for_stream_state(doc, state_type, current_state): return next_states +@login_required +def offer_wg_action_helpers(request, name): + doc = get_object_or_404(Document, type="draft", name=name) + if doc.stream is None or doc.stream_id != "ietf" or doc.became_rfc() is not None: + raise Http404 + + if not is_authorized_in_doc_stream(request.user, doc): + permission_denied(request, "You don't have permission to access this page.") + + return render(request, "doc/draft/wg_action_helpers.html", + {"doc": doc, + }) + @login_required def change_stream_state(request, name, state_type): doc = get_object_or_404(Document, type="draft", name=name) @@ -1736,10 +1746,17 @@ def change_stream_state(request, name, state_type): prev_state = doc.get_state(state_type.slug) next_states = next_states_for_stream_state(doc, state_type, prev_state) + # These tell the form to allow directly setting the state to fix up errors. can_set_sub_pub = has_role(request.user,('Secretariat','Area Director')) or (prev_state and prev_state.slug=='sub-pub') if request.method == 'POST': - form = ChangeStreamStateForm(request.POST, doc=doc, state_type=state_type,can_set_sub_pub=can_set_sub_pub,stream=doc.stream) + form = ChangeStreamStateForm( + request.POST, + doc=doc, + state_type=state_type, + can_set_sub_pub=can_set_sub_pub, + stream=doc.stream, + ) if form.is_valid(): by = request.user.person events = [] @@ -1754,13 +1771,13 @@ def change_stream_state(request, name, state_type): events.append(e) due_date = None - if form.cleaned_data["weeks"] != None: + if form.cleaned_data["weeks"] is not None: due_date = datetime_today(DEADLINE_TZINFO) + datetime.timedelta(weeks=form.cleaned_data["weeks"]) update_reminder(doc, "stream-s", e, due_date) email_stream_state_changed(request, doc, prev_state, new_state, by, comment) - + # tags existing_tags = set(doc.tags.all()) new_tags = set(form.cleaned_data["tags"]) @@ -1796,8 +1813,15 @@ def change_stream_state(request, name, state_type): else: form.add_error(None, "No change in state or tags found, and no comment provided -- nothing to do.") else: - form = ChangeStreamStateForm(initial=dict(new_state=prev_state.pk if prev_state else None, tags= doc.tags.all()), - doc=doc, state_type=state_type, can_set_sub_pub = can_set_sub_pub,stream = doc.stream) + form = ChangeStreamStateForm( + initial=dict( + new_state=prev_state.pk if prev_state else None, tags=doc.tags.all() + ), + doc=doc, + state_type=state_type, + can_set_sub_pub=can_set_sub_pub, + stream=doc.stream, + ) milestones = doc.groupmilestone_set.all() @@ -1842,3 +1866,325 @@ def set_intended_status_level(request, doc, new_level, old_level, comment): msg = "\n".join(e.desc for e in events) email_intended_status_changed(request, doc, msg) + +class IssueWorkingGroupLastCallForm(forms.Form): + end_date = DatepickerDateField( + required=True, + date_format="yyyy-mm-dd", + picker_settings={ + "autoclose": "1", + }, + help_text="The date the Last Call closes. If you change this, review the subject and body carefully to ensure the change is captured correctly.", + ) + + to = MultiEmailField( + required=True, + help_text="Comma separated list of address to use in the To: header", + ) + cc = MultiEmailField( + required=False, help_text="Comma separated list of addresses to copy" + ) + subject = forms.CharField( + required=True, + help_text="Subject for Last Call message. If you change the date here, be sure to make a matching change in the body.", + ) + body = forms.CharField( + widget=forms.Textarea, required=True, help_text="Body for Last Call message" + ) + + def clean_end_date(self): + end_date = self.cleaned_data["end_date"] + if end_date <= date_today(DEADLINE_TZINFO): + raise forms.ValidationError("End date must be later than today") + return end_date + + def clean(self): + cleaned_data = super().clean() + end_date = cleaned_data.get("end_date") + if end_date is not None: + body = cleaned_data.get("body") + subject = cleaned_data.get("subject") + if end_date.isoformat() not in body: + self.add_error( + "body", + forms.ValidationError( + f"Last call end date ({end_date.isoformat()}) not found in body" + ), + ) + if end_date.isoformat() not in subject: + self.add_error( + "subject", + forms.ValidationError( + f"Last call end date ({end_date.isoformat()}) not found in subject" + ), + ) + return cleaned_data + + +@login_required +def issue_wg_lc(request, name): + doc = get_object_or_404(Document, name=name) + + if doc.stream_id != "ietf": + raise Http404 + if doc.type_id != "draft" or doc.group.type_id != "wg": + raise Http404 + if doc.get_state_slug("draft-stream-ietf") == "wg-lc": + raise Http404 + if doc.get_state_slug("draft") == "rfc": + raise Http404 + + if not is_authorized_in_doc_stream(request.user, doc): + permission_denied(request, "You don't have permission to access this page.") + + if request.method == "POST": + form = IssueWorkingGroupLastCallForm(request.POST) + if form.is_valid(): + # Intentionally not changing tags or adding a comment + # those things can be done with other workflows + by = request.user.person + prev_state = doc.get_state("draft-stream-ietf") + events = [] + wglc_state = State.objects.get(type="draft-stream-ietf", slug="wg-lc") + doc.set_state(wglc_state) + e = add_state_change_event(doc, by, prev_state, wglc_state) + events.append(e) + end_date = form.cleaned_data["end_date"] + update_reminder( + doc, "stream-s", e, datetime_from_date(end_date, DEADLINE_TZINFO) + ) + doc.save_with_history(events) + email_stream_state_changed(request, doc, prev_state, wglc_state, by) + send_mail_text( + request, + to = form.cleaned_data["to"], + frm = request.user.person.formatted_email(), + subject = form.cleaned_data["subject"], + txt = form.cleaned_data["body"], + cc = form.cleaned_data["cc"], + ) + return redirect("ietf.doc.views_doc.document_main", name=doc.name) + else: + end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=14) + subject = f"WG Last Call: {doc.name}-{doc.rev} (Ends {end_date})" + body = render_to_string( + "doc/mail/wg_last_call_issued.txt", + dict( + doc=doc, + end_date=end_date, + wg_list=doc.group.list_email, + settings=settings, + ), + ) + (to, cc) = gather_address_lists("doc_wg_last_call_issued", doc=doc) + + form = IssueWorkingGroupLastCallForm( + initial=dict( + end_date=end_date, + to=", ".join(to), + cc=", ".join(cc), + subject=subject, + body=body, + ) + ) + + return render( + request, + "doc/draft/issue_working_group_last_call.html", + dict( + doc=doc, + form=form, + ), + ) + +class IssueCallForAdoptionForm(forms.Form): + end_date = DatepickerDateField( + required=True, + date_format="yyyy-mm-dd", + picker_settings={ + "autoclose": "1", + }, + help_text="The date the Call for Adoption closes. If you change this, review the subject and body carefully to ensure the change is captured correctly.", + ) + + to = MultiEmailField( + required=True, + help_text="Comma separated list of address to use in the To: header", + ) + cc = MultiEmailField( + required=False, help_text="Comma separated list of addresses to copy" + ) + subject = forms.CharField( + required=True, + help_text="Subject for Call for Adoption message. If you change the date here, be sure to make a matching change in the body.", + ) + body = forms.CharField( + widget=forms.Textarea, required=True, help_text="Body for Call for Adoption message" + ) + + def clean_end_date(self): + end_date = self.cleaned_data["end_date"] + if end_date <= date_today(DEADLINE_TZINFO): + raise forms.ValidationError("End date must be later than today") + return end_date + + def clean(self): + cleaned_data = super().clean() + end_date = cleaned_data.get("end_date") + if end_date is not None: + body = cleaned_data.get("body") + subject = cleaned_data.get("subject") + if end_date.isoformat() not in body: + self.add_error( + "body", + forms.ValidationError( + f"Call for adoption end date ({end_date.isoformat()}) not found in body" + ), + ) + if end_date.isoformat() not in subject: + self.add_error( + "subject", + forms.ValidationError( + f"Call for adoption end date ({end_date.isoformat()}) not found in subject" + ), + ) + return cleaned_data + +@login_required +def issue_wg_call_for_adoption(request, name, acronym): + doc = get_object_or_404(Document, name=name) + group = Group.objects.filter(acronym=acronym, type_id="wg").first() + reject = False + if group is None or doc.type_id != "draft" or not is_doc_ietf_adoptable(doc): + reject = True + if doc.stream is None: + if not can_adopt_draft(request.user, doc): + reject = True + elif doc.stream_id != "ietf": + reject = True + else: # doc.stream_id == "ietf" + if not is_authorized_in_doc_stream(request.user, doc): + reject = True + if reject: + raise permission_denied(request, f"You can't issue a {acronym} wg call for adoption for this document.") + + if request.method == "POST": + form = IssueCallForAdoptionForm(request.POST) + if form.is_valid(): + # Intentionally not changing tags or adding a comment + # those things can be done with other workflows + by = request.user.person + + events = [] + if doc.stream_id != "ietf": + stream = StreamName.objects.get(slug="ietf") + doc.stream = stream + e = DocEvent(type="changed_stream", doc=doc, rev=doc.rev, by=by) + e.desc = f"Changed stream to {stream.name}" # Propogates embedding html in DocEvent.desc for consistency + e.save() + events.append(e) + if doc.group != group: + doc.group = group + e = DocEvent(type="changed_group", doc=doc, rev=doc.rev, by=by) + e.desc = f"Changed group to {group.name} ({group.acronym.upper()})" # Even if it makes the cats cry + e.save() + events.append(e) + prev_state = doc.get_state("draft-stream-ietf") + c_adopt_state = State.objects.get(type="draft-stream-ietf", slug="c-adopt") + doc.set_state(c_adopt_state) + e = add_state_change_event(doc, by, prev_state, c_adopt_state) + events.append(e) + end_date = form.cleaned_data["end_date"] + update_reminder( + doc, "stream-s", e, datetime_from_date(end_date, DEADLINE_TZINFO) + ) + doc.save_with_history(events) + email_stream_state_changed(request, doc, prev_state, c_adopt_state, by) + send_mail_text( + request, + to = form.cleaned_data["to"], + frm = request.user.person.formatted_email(), + subject = form.cleaned_data["subject"], + txt = form.cleaned_data["body"], + cc = form.cleaned_data["cc"], + ) + return redirect("ietf.doc.views_doc.document_main", name=doc.name) + else: + end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=14) + subject = f"Call for adoption: {doc.name}-{doc.rev} (Ends {end_date})" + body = render_to_string( + "doc/mail/wg_call_for_adoption_issued.txt", + dict( + doc=doc, + group=group, + end_date=end_date, + wg_list=doc.group.list_email, + settings=settings, + ), + ) + (to, cc) = gather_address_lists("doc_wg_call_for_adoption_issued", doc=doc) + if doc.group.acronym == "none": + to.insert(0, f"{group.acronym}-chairs@ietf.org") + to.insert(0, group.list_email) + form = IssueCallForAdoptionForm( + initial=dict( + end_date=end_date, + to=", ".join(to), + cc=", ".join(cc), + subject=subject, + body=body, + ) + ) + + return render( + request, + "doc/draft/issue_working_group_call_for_adoption.html", + dict( + doc=doc, + form=form, + ), + ) + +class GroupModelChoiceField(forms.ModelChoiceField): + def label_from_instance(self, obj): + return f"{obj.acronym} - {obj.name}" + + +class WgForm(forms.Form): + group = GroupModelChoiceField( + queryset=Group.objects.filter(type_id="wg", state="active") + .order_by("acronym") + .distinct(), + required=True, + empty_label="Select IETF Working Group", + ) + + def __init__(self, *args, **kwargs): + user = kwargs.pop("user") + super(WgForm, self).__init__(*args, **kwargs) + if not has_role(user, ["Secretariat", "Area Director"]): + self.fields["group"].queryset = self.fields["group"].queryset.filter( + role__name_id="chair", role__person=user.person + ) + + +@role_required("Secretariat", "WG Chair") +def ask_about_ietf_adoption_call(request, name): + doc = get_object_or_404(Document, name=name) + if doc.stream is not None or doc.group.acronym != "none": + raise Http404 + if request.method == "POST": + form = WgForm(request.POST, user=request.user) + if form.is_valid(): + group = form.cleaned_data["group"] + return redirect(issue_wg_call_for_adoption, name=doc.name, acronym=group.acronym) + else: + form = WgForm(initial={"group": None}, user=request.user) + return render( + request, + "doc/draft/ask_about_ietf_adoption.html", + dict( + doc=doc, + form=form, + ), + ) diff --git a/ietf/doc/views_material.py b/ietf/doc/views_material.py index 6f8b8a8f12..eefac0ca61 100644 --- a/ietf/doc/views_material.py +++ b/ietf/doc/views_material.py @@ -22,6 +22,7 @@ from ietf.doc.utils import add_state_change_event, check_common_doc_name_rules from ietf.group.models import Group from ietf.group.utils import can_manage_materials +from ietf.meeting.utils import resolve_uploaded_material from ietf.utils import log from ietf.utils.decorators import ignore_view_kwargs from ietf.utils.meetecho import MeetechoAPIError, SlidesManager @@ -179,6 +180,9 @@ def edit_material(request, name=None, acronym=None, action=None, doc_type=None): "There was an error creating a hardlink at %s pointing to %s: %s" % (ftp_filepath, filepath, ex) ) + else: + for meeting in set([s.meeting for s in doc.session_set.all()]): + resolve_uploaded_material(meeting=meeting, doc=doc) if prev_rev != doc.rev: e = NewRevisionDocEvent(type="new_revision", doc=doc, rev=doc.rev) diff --git a/ietf/doc/views_review.py b/ietf/doc/views_review.py index 1f23c435fa..1968b133ce 100644 --- a/ietf/doc/views_review.py +++ b/ietf/doc/views_review.py @@ -897,7 +897,7 @@ def complete_review(request, name, assignment_id=None, acronym=None): if need_to_email_review: # email the review - subject = "{} {} {} of {}-{}".format(assignment.review_request.team.acronym.capitalize(),assignment.review_request.type.name.lower(),"partial review" if assignment.state_id == "part-completed" else "review", assignment.review_request.doc.name, assignment.reviewed_rev) + subject = "{}-{} {} {} {}".format(assignment.review_request.doc.name, assignment.reviewed_rev, assignment.review_request.type.name.lower(), assignment.review_request.team.acronym.capitalize(), "partial review" if assignment.state_id == "part-completed" else "review") related_groups = [ assignment.review_request.team, ] if assignment.review_request.doc.group: related_groups.append(assignment.review_request.doc.group) @@ -958,14 +958,14 @@ def complete_review(request, name, assignment_id=None, acronym=None): form = CompleteReviewForm(assignment, doc, team, is_reviewer, initial=initial) - mail_archive_query_urls = mailarch.construct_query_urls(doc, team) + mail_archive_query_data = mailarch.construct_query_data(doc, team) return render(request, 'doc/review/complete_review.html', { 'doc': doc, 'team': team, 'assignment': assignment, 'form': form, - 'mail_archive_query_urls': mail_archive_query_urls, + 'mail_archive_query_data': mail_archive_query_data, 'revising_review': revising_review, 'review_to': to, 'review_cc': cc, @@ -987,27 +987,25 @@ def search_mail_archive(request, name, acronym=None, assignment_id=None): if not (is_reviewer or can_manage_request): permission_denied(request, "You do not have permission to perform this action") - res = mailarch.construct_query_urls(doc, team, query=request.GET.get("query")) - if not res: - return JsonResponse({ "error": "Couldn't do lookup in mail archive - don't know where to look"}) - - MAX_RESULTS = 30 + query_data = mailarch.construct_query_data(doc, team, query=request.GET.get("query")) + if not query_data: + return JsonResponse({"error": "Couldn't do lookup in mail archive - don't know where to look"}) try: - res["messages"] = mailarch.retrieve_messages(res["query_data_url"])[:MAX_RESULTS] - for message in res["messages"]: + query_data["messages"] = mailarch.retrieve_messages(query_data) + for message in query_data["messages"]: try: revision_guess = message["subject"].split(name)[1].split('-')[1] message["revision_guess"] = revision_guess if revision_guess.isnumeric() else None except IndexError: pass except KeyError as e: - res["error"] = "No results found (%s)" % str(e) + query_data["error"] = "No results found (%s)" % str(e) except Exception as e: - res["error"] = "Retrieval from mail archive failed: %s" % str(e) + query_data["error"] = "Retrieval from mail archive failed: %s" % str(e) # raise # useful when debugging - return JsonResponse(res) + return JsonResponse(query_data) class EditReviewRequestCommentForm(forms.ModelForm): comment = forms.CharField(widget=forms.Textarea, strip=False) diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index 9e9b5e88dd..4232d77f6c 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -59,7 +59,7 @@ import debug # pyflakes:ignore from ietf.doc.models import ( Document, DocHistory, State, - LastCallDocEvent, NewRevisionDocEvent, IESG_SUBSTATE_TAGS, + NewRevisionDocEvent, IESG_SUBSTATE_TAGS, IESG_BALLOT_ACTIVE_STATES, IESG_STATCHG_CONFLREV_ACTIVE_STATES, IESG_CHARTER_ACTIVE_STATES ) from ietf.doc.fields import select2_id_doc_name_json @@ -74,7 +74,7 @@ from ietf.utils.log import log from ietf.doc.utils_search import prepare_document_table, doc_type, doc_state, doc_type_name, AD_WORKLOAD from ietf.ietfauth.utils import has_role - +from ietf.utils.unicodenormalize import normalize_for_sorting class SearchForm(forms.Form): name = forms.CharField(required=False) @@ -219,7 +219,7 @@ def retrieve_search_results(form, all_types=False): queries.extend([Q(targets_related__source__name__icontains=look_for, targets_related__relationship_id="became_rfc")]) combined_query = reduce(operator.or_, queries) - docs = docs.filter(combined_query).distinct() + docs = docs.filter(combined_query) # rfc/active/old check buttons allowed_draft_states = [] @@ -229,20 +229,23 @@ def retrieve_search_results(form, all_types=False): allowed_draft_states.extend(['repl', 'expired', 'auth-rm', 'ietf-rm']) docs = docs.filter(Q(states__slug__in=allowed_draft_states) | - ~Q(type__slug='draft')).distinct() + ~Q(type__slug='draft')) # radio choices by = query["by"] if by == "author": docs = docs.filter( Q(documentauthor__person__alias__name__icontains=query["author"]) | - Q(documentauthor__person__email__address__icontains=query["author"]) + Q(documentauthor__person__email__address__icontains=query["author"]) | + Q(rfcauthor__person__alias__name__icontains=query["author"]) | + Q(rfcauthor__person__email__address__icontains=query["author"]) | + Q(rfcauthor__titlepage_name__icontains=query["author"]) ) elif by == "group": docs = docs.filter(group__acronym__iexact=query["group"]) elif by == "area": docs = docs.filter(Q(group__type="wg", group__parent=query["area"]) | - Q(group=query["area"])).distinct() + Q(group=query["area"])) elif by == "ad": docs = docs.filter(ad=query["ad"]) elif by == "state": @@ -255,6 +258,8 @@ def retrieve_search_results(form, all_types=False): elif by == "stream": docs = docs.filter(stream=query["stream"]) + docs=docs.distinct() + return docs @@ -480,6 +485,7 @@ def _state_to_doc_type(state): ).distinct(): if p in get_active_ads(): ads.append(p) + ads.sort(key=lambda p: normalize_for_sorting(p.plain_name())) bucket_template = { dt: {state: [[] for _ in range(days)] for state in STATE_SLUGS[dt].values()} @@ -665,6 +671,7 @@ def sort_key(doc): r.get_state_slug("draft-iesg") == "dead" or r.get_state_slug("draft") == "repl" or r.get_state_slug("draft") == "rfc" + or (r.get_state_slug("draft") == "expired" and r.get_state_slug("draft-iesg") == "idexists") ) ) ] @@ -752,6 +759,90 @@ def sort_key(doc): ) +def docs_for_iesg(request): + def sort_key(doc): + dt = doc_type(doc) + dt_key = list(AD_WORKLOAD.keys()).index(dt) + ds = doc_state(doc) + ds_key = AD_WORKLOAD[dt].index(ds) if ds in AD_WORKLOAD[dt] else 99 + return dt_key * 100 + ds_key + + results, meta = prepare_document_table( + request, + Document.objects.filter( + ad__in=Person.objects.filter( + Q( + role__name__in=("pre-ad", "ad"), + role__group__type="area", + role__group__state="active", + ) + ) + ).exclude( + type_id="rfc", + ).exclude( + type_id="draft", + states__type="draft", + states__slug__in=["repl", "rfc"], + ).exclude( + type_id="draft", + states__type="draft-iesg", + states__slug__in=["idexists", "rfcqueue"], + ).exclude( + type_id="conflrev", + states__type="conflrev", + states__slug__in=["appr-noprob-sent", "appr-reqnopub-sent", "withdraw", "dead"], + ).exclude( + type_id="statchg", + states__type="statchg", + states__slug__in=["appr-sent", "dead"], + ).exclude( + type_id="charter", + states__type="charter", + states__slug__in=["notrev", "infrev", "approved", "replaced"], + ), + max_results=1000, + show_ad_and_shepherd=True, + ) + results.sort(key=lambda d: sort_key(d)) + + # filter out some results + results = [ + r + for r in results + if not ( + r.type_id == "charter" + and ( + r.group.state_id == "abandon" + or r.get_state_slug("charter") == "replaced" + ) + ) + and not ( + r.type_id == "draft" + and ( + r.get_state_slug("draft-iesg") == "dead" + or r.get_state_slug("draft") == "repl" + or r.get_state_slug("draft") == "rfc" + ) + ) + ] + + _calculate_state_name = get_state_name_calculator() + for d in results: + dt = d.type.slug + d.search_heading = _calculate_state_name(dt, doc_state(d)) + if d.search_heading != "RFC": + d.search_heading += f" {doc_type_name(dt)}" + + return render( + request, + "doc/drafts_for_iesg.html", + { + "docs": results, + "meta": meta, + }, + ) + + def drafts_in_last_call(request): lc_state = State.objects.get(type="draft-iesg", slug="lc").pk form = SearchForm({'by':'state','state': lc_state, 'rfcs':'on', 'activedrafts':'on'}) @@ -764,31 +855,6 @@ def drafts_in_last_call(request): 'form':form, 'docs':results, 'meta':meta, 'pages':pages }) -def drafts_in_iesg_process(request): - states = State.objects.filter(type="draft-iesg").exclude(slug__in=('idexists', 'pub', 'dead', 'rfcqueue')) - title = "Documents in IESG process" - - grouped_docs = [] - - for s in states.order_by("order"): - docs = Document.objects.filter(type="draft", states=s).distinct().order_by("time").select_related("ad", "group", "group__parent") - if docs: - if s.slug == "lc": - for d in docs: - e = d.latest_event(LastCallDocEvent, type="sent_last_call") - # If we don't have an event, use an arbitrary date in the past (but not datetime.datetime.min, - # which causes problems with timezone conversions) - d.lc_expires = e.expires if e else datetime.datetime(1950, 1, 1) - docs = list(docs) - docs.sort(key=lambda d: d.lc_expires) - - grouped_docs.append((s, docs)) - - return render(request, 'doc/drafts_in_iesg_process.html', { - "grouped_docs": grouped_docs, - "title": title, - }) - def recent_drafts(request, days=7): slowcache = caches['slowpages'] cache_key = f'recentdraftsview{days}' diff --git a/ietf/doc/views_statement.py b/ietf/doc/views_statement.py index 9dc8c8ad69..4b85c81d83 100644 --- a/ietf/doc/views_statement.py +++ b/ietf/doc/views_statement.py @@ -1,4 +1,5 @@ -# Copyright The IETF Trust 2023, All Rights Reserved +# Copyright The IETF Trust 2023-2025, All Rights Reserved +from django.contrib import messages import debug # pyflakes: ignore @@ -6,10 +7,13 @@ from django import forms from django.conf import settings -from django.http import FileResponse, Http404 +from django.http import FileResponse, Http404, HttpResponseRedirect from django.views.decorators.cache import cache_control from django.shortcuts import get_object_or_404, render, redirect from django.template.loader import render_to_string + +from ietf.doc.forms import ChangeStatementStateForm +from ietf.doc.utils import add_state_change_event from ietf.utils import markdown from django.utils.html import escape @@ -278,3 +282,40 @@ def new_statement(request): } form = NewStatementForm(initial=init) return render(request, "doc/statement/new_statement.html", {"form": form}) + + +@role_required("Secretariat") +def change_statement_state(request, name): + """Change state of a statement Document""" + statement = get_object_or_404( + Document.objects.filter(type_id="statement"), + name=name, + ) + if request.method == "POST": + form = ChangeStatementStateForm(request.POST) + if form.is_valid(): + new_state = form.cleaned_data["state"] + prev_state = statement.get_state() + if new_state == prev_state: + messages.info(request, f"State not changed, remains {prev_state}.") + else: + statement.set_state(new_state) + e = add_state_change_event( + statement, + request.user.person, + prev_state, + new_state, + ) + statement.save_with_history([e]) + messages.success(request, f"State changed to {new_state}.") + return HttpResponseRedirect(statement.get_absolute_url()) + else: + form = ChangeStatementStateForm(initial={"state": statement.get_state()}) + return render( + request, + "doc/statement/change_statement_state.html", + { + "form": form, + "statement": statement, + }, + ) diff --git a/ietf/doc/views_stats.py b/ietf/doc/views_stats.py index 0bbf0b91c5..028573b338 100644 --- a/ietf/doc/views_stats.py +++ b/ietf/doc/views_stats.py @@ -18,7 +18,7 @@ from ietf.utils.timezone import date_today -epochday = datetime.datetime.utcfromtimestamp(0).date().toordinal() +epochday = datetime.datetime.fromtimestamp(0, datetime.UTC).date().toordinal() def dt(s): @@ -35,13 +35,13 @@ def model_to_timeline_data(model, field='time', **kwargs): assert field in [ f.name for f in model._meta.get_fields() ] objects = ( model.objects.filter(**kwargs) - .annotate(date=TruncDate(field, tzinfo=datetime.timezone.utc)) + .annotate(date=TruncDate(field, tzinfo=datetime.UTC)) .order_by('date') .values('date') .annotate(count=Count('id'))) if objects.exists(): obj_list = list(objects) - today = date_today(datetime.timezone.utc) + today = date_today(datetime.UTC) if not obj_list[-1]['date'] == today: obj_list += [ {'date': today, 'count': 0} ] data = [ ((e['date'].toordinal()-epochday)*1000*60*60*24, e['count']) for e in obj_list ] diff --git a/ietf/group/admin.py b/ietf/group/admin.py index fedec49d85..685c10aeea 100644 --- a/ietf/group/admin.py +++ b/ietf/group/admin.py @@ -26,14 +26,15 @@ MilestoneGroupEvent, GroupExtResource, Appeal, AppealArtifact ) from ietf.name.models import GroupTypeName -from ietf.utils.validators import validate_external_resource_value +from ietf.utils.admin import SaferTabularInline from ietf.utils.response import permission_denied +from ietf.utils.validators import validate_external_resource_value -class RoleInline(admin.TabularInline): +class RoleInline(SaferTabularInline): model = Role raw_id_fields = ["person", "email"] -class GroupURLInline(admin.TabularInline): +class GroupURLInline(SaferTabularInline): model = GroupURL class GroupForm(forms.ModelForm): diff --git a/ietf/group/migrations/0005_remove_sdo_authorized_individuals.py b/ietf/group/migrations/0005_remove_sdo_authorized_individuals.py new file mode 100644 index 0000000000..77fe25b467 --- /dev/null +++ b/ietf/group/migrations/0005_remove_sdo_authorized_individuals.py @@ -0,0 +1,192 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from collections import defaultdict + +from django.db import migrations + +from ietf.person.name import plain_name + + +def get_plain_name(person): + return person.plain or plain_name(person.name) + + +def forward(apps, schema_editor): + """Remove any 'auth' Role objects for groups of type 'sdo' + + The IAB has decided that the Authorized Individual concept for + authorizing entry or management of liaison statments hasn't worked + well - the roles for the groups are not being maintained, Instead, + the concept will be removed and the liaison managers or secretariat + (and soon the liaison coordinators) will operate the liaison tool + on their behalf. + """ + Role = apps.get_model("group", "Role") + GroupEvent = apps.get_model("group", "GroupEvent") + groups = defaultdict(list) + role_qs = Role.objects.filter(name_id="auth", group__type_id="sdo") + for role in role_qs: + groups[role.group].append(role) + for group in groups: + desc = f"Removed Authorized Persons: {', '.join([get_plain_name(role.person) for role in groups[group]])}" + GroupEvent.objects.create( + group=group, + by_id=1, # (System) + desc=desc, + ) + role_qs.delete() + + +def reverse(apps, schema_editor): + """Intentionally does nothing""" + pass + + +class Migration(migrations.Migration): + dependencies = [ + ("group", "0004_modern_list_archive"), + ] + + operations = [migrations.RunPython(forward, reverse)] + + +# At the time this migration was created, it would have removed these Role objects: +# { "authorized_individuals" : [ +# {"person_id": 107937, "group_id": 56, "email": "hannu.hietalahti@nokia.com" }, # Hannu Hietalahti is Authorized Individual in 3gpp +# {"person_id": 107943, "group_id": 56, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Authorized Individual in 3gpp +# {"person_id": 112807, "group_id": 56, "email": "Paolo.Usai@etsi.org" }, # Paolo Usai is Authorized Individual in 3gpp +# {"person_id": 105859, "group_id": 56, "email": "atle.monrad@ericsson.com" }, # Atle Monrad is Authorized Individual in 3gpp +# {"person_id": 116149, "group_id": 1907, "email": "tsgsx_chair@3GPP2.org" }, # Xiaowu Zhao is Authorized Individual in 3gpp2-tsg-sx +# {"person_id": 120914, "group_id": 1902, "email": "ozgur.oyman@intel.com" }, # Ozgur Oyman is Authorized Individual in 3gpp-tsgsa-sa4 +# {"person_id": 107943, "group_id": 1902, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Authorized Individual in 3gpp-tsgsa-sa4 +# {"person_id": 119203, "group_id": 1902, "email": "fanyanping@huawei.com" }, # Yanping Fan is Authorized Individual in 3gpp-tsgsa-sa4 +# {"person_id": 112977, "group_id": 1902, "email": "tomas.frankkila@ericsson.com" }, # Tomas Frankkila is Authorized Individual in 3gpp-tsgsa-sa4 +# {"person_id": 120240, "group_id": 2019, "email": "CM8655@att.com" }, # Peter Musgrove is Authorized Individual in atis-eloc-tf +# {"person_id": 120241, "group_id": 2019, "email": "Christian.Militeau@intrado.com" }, # Christian Militeau is Authorized Individual in atis-eloc-tf +# {"person_id": 120243, "group_id": 2019, "email": "ablasgen@atis.org" }, # Alexandra Blasgen is Authorized Individual in atis-eloc-tf +# {"person_id": 114696, "group_id": 67, "email": "KEN.KO@adtran.com" }, # Ken Ko is Authorized Individual in broadband-forum +# {"person_id": 119494, "group_id": 67, "email": "michael.fargano@centurylink.com" }, # Michael Fargano is Authorized Individual in broadband-forum +# {"person_id": 124318, "group_id": 67, "email": "joey.boyd@adtran.com" }, # Joey Boyd is Authorized Individual in broadband-forum +# {"person_id": 114762, "group_id": 67, "email": "bwelch@juniper.net" }, # Bill Welch is Authorized Individual in broadband-forum +# {"person_id": 112837, "group_id": 67, "email": "christophe.alter@orange.com" }, # Christophe Alter is Authorized Individual in broadband-forum +# {"person_id": 141083, "group_id": 2407, "email": "dan.middleton@intel.com" }, # Dan Middleton is Authorized Individual in confidential-computing-consortium +# {"person_id": 117421, "group_id": 1933, "email": "chairman@dmtf.org" }, # Winston Bumpus is Authorized Individual in dmtf +# {"person_id": 116529, "group_id": 1919, "email": "istvan@ecma-international.org" }, # Istvan Sebestyen is Authorized Individual in ecma-tc39 +# {"person_id": 116363, "group_id": 1915, "email": "e2nasupport@etsi.org" }, # Sonia Compans is Authorized Individual in etsi-e2na +# {"person_id": 116862, "group_id": 2003, "email": "latif@ladid.lu" }, # Latif Ladid is Authorized Individual in etsi-isg-ip6 +# {"person_id": 116283, "group_id": 2198, "email": "adrian.neal@vodafone.com" }, # Adrian Neal is Authorized Individual in etsi-isg-mec +# {"person_id": 119412, "group_id": 2004, "email": "jkfernic@uwaterloo.ca" }, # Jennifer Fernick is Authorized Individual in etsi-isg-qsc +# {"person_id": 122406, "group_id": 2165, "email": "d.lake@surrey.ac.uk" }, # David Lake is Authorized Individual in etsi-ngp +# {"person_id": 122407, "group_id": 2165, "email": "andy.sutton@ee.co.uk" }, # Andy Sutton is Authorized Individual in etsi-ngp +# {"person_id": 112609, "group_id": 2165, "email": "richard.li@futurewei.com" }, # Richard Li is Authorized Individual in etsi-ngp +# {"person_id": 122406, "group_id": 2177, "email": "d.lake@surrey.ac.uk" }, # David Lake is Authorized Individual in etsi-ngp-isp +# {"person_id": 112609, "group_id": 2177, "email": "richard.li@futurewei.com" }, # Richard Li is Authorized Individual in etsi-ngp-isp +# {"person_id": 122407, "group_id": 2177, "email": "andy.sutton@ee.co.uk" }, # Andy Sutton is Authorized Individual in etsi-ngp-isp +# {"person_id": 118527, "group_id": 1986, "email": "luca.pesando@telecomitalia.it" }, # Luca Pesando is Authorized Individual in etsi-ntech +# {"person_id": 118526, "group_id": 1986, "email": "NTECHsupport@etsi.org" }, # Sylwia Korycinska is Authorized Individual in etsi-ntech +# {"person_id": 116052, "group_id": 1904, "email": "Beniamino.gorini@alcatel-lucent.com" }, # Gorini Beniamino is Authorized Individual in etsi-tc-ee +# {"person_id": 19651, "group_id": 63, "email": "glenn.parsons@ericsson.com" }, # Glenn Parsons is Authorized Individual in ieee-802-1 +# {"person_id": 107599, "group_id": 63, "email": "tony@jeffree.co.uk" }, # Tony Jeffree is Authorized Individual in ieee-802-1 +# {"person_id": 117415, "group_id": 1862, "email": "Adrian.P.Stephens@intel.com" }, # Adrian Stephens is Authorized Individual in ieee-802-11 +# {"person_id": 106284, "group_id": 1862, "email": "dstanley@arubanetworks.com" }, # Dorothy Stanley is Authorized Individual in ieee-802-11 +# {"person_id": 114106, "group_id": 1871, "email": "r.b.marks@ieee.org" }, # Roger Marks is Authorized Individual in ieee-802-16 +# {"person_id": 101753, "group_id": 1885, "email": "max.riegel@ieee.org" }, # Max Riegel is Authorized Individual in ieee-802-ec-omniran +# {"person_id": 113810, "group_id": 1859, "email": "jehrig@inventures.com" }, # John Ehrig is Authorized Individual in imtc +# {"person_id": 123010, "group_id": 48, "email": "Emil.Kowalczyk@orange.com" }, # Emil Kowalczyk is Authorized Individual in iso-iec-jtc1-sc2 +# {"person_id": 11182, "group_id": 48, "email": "paf@netnod.se" }, # Patrik Fältström is Authorized Individual in iso-iec-jtc1-sc2 +# {"person_id": 117429, "group_id": 1939, "email": "krystyna.passia@din.de" }, # Krystyna Passia is Authorized Individual in iso-iec-jtc1-sc27 +# {"person_id": 117428, "group_id": 1939, "email": "walter.fumy@bdr.de" }, # Walter Fumy is Authorized Individual in iso-iec-jtc1-sc27 +# {"person_id": 114435, "group_id": 74, "email": "watanabe@itscj.ipsj.or.jp" }, # Shinji Watanabe is Authorized Individual in iso-iec-jtc1-sc29-wg11 +# {"person_id": 112106, "group_id": 49, "email": "jooran@kisi.or.kr" }, # Jooran Lee is Authorized Individual in iso-iec-jtc1-sc6 +# {"person_id": 17037, "group_id": 49, "email": "dykim@comsun.chungnnam.ac.kr" }, # Dae Kim is Authorized Individual in iso-iec-jtc1-sc6 +# {"person_id": 117426, "group_id": 1938, "email": "chair@jtc1-sc7.org" }, # Francois Coallier is Authorized Individual in iso-iec-jtc1-sc7 +# {"person_id": 117427, "group_id": 1938, "email": "secretariat@jtc1-sc7.org" }, # Witold Suryn is Authorized Individual in iso-iec-jtc1-sc7 +# {"person_id": 118769, "group_id": 2144, "email": "alexandre.petrescu@gmail.com" }, # Alexandre Petrescu is Authorized Individual in isotc204 +# {"person_id": 115544, "group_id": 1890, "email": "sergio.buonomo@itu.int" }, # Sergio Buonomo is Authorized Individual in itu-r +# {"person_id": 122111, "group_id": 2157, "email": "h.mazar@atdi.com" }, # Haim Mazar is Authorized Individual in itu-r-wp-5c +# {"person_id": 115544, "group_id": 2157, "email": "sergio.buonomo@itu.int" }, # Sergio Buonomo is Authorized Individual in itu-r-wp-5c +# {"person_id": 112105, "group_id": 51, "email": "Malcolm.Johnson@itu.int" }, # Malcom Johnson is Authorized Individual in itu-t +# {"person_id": 113911, "group_id": 1860, "email": "martin.adolph@itu.int" }, # Martin Adolph is Authorized Individual in itu-t-fg-dist +# {"person_id": 122779, "group_id": 2180, "email": "Leo.Lehmann@bakom.admin.ch" }, # Leo Lehmann is Authorized Individual in itu-t-fg-imt-2020 +# {"person_id": 103383, "group_id": 2180, "email": "peter.ashwoodsmith@huawei.com" }, # Peter Ashwood-Smith is Authorized Individual in itu-t-fg-imt-2020 +# {"person_id": 107300, "group_id": 1872, "email": "tatiana.kurakova@itu.int" }, # Tatiana Kurakova is Authorized Individual in itu-t-jca-cloud +# {"person_id": 106224, "group_id": 1872, "email": "mmorrow@cisco.com" }, # Monique Morrow is Authorized Individual in itu-t-jca-cloud +# {"person_id": 105714, "group_id": 1874, "email": "martin.euchner@itu.int" }, # Martin Euchner is Authorized Individual in itu-t-jca-cop +# {"person_id": 106475, "group_id": 2170, "email": "khj@etri.re.kr" }, # Hyoung-Jun Kim is Authorized Individual in itu-t-jca-iot-scc +# {"person_id": 122491, "group_id": 2170, "email": "tsbjcaiot@itu.int" }, # ITU Tsb is Authorized Individual in itu-t-jca-iot-scc +# {"person_id": 122490, "group_id": 2170, "email": "fabio.bigi@virgilio.it" }, # Fabio Bigi is Authorized Individual in itu-t-jca-iot-scc +# {"person_id": 116952, "group_id": 1927, "email": "chengying10@chinaunicom.cn" }, # Ying Cheng is Authorized Individual in itu-t-jca-sdn +# {"person_id": 111205, "group_id": 1927, "email": "t-egawa@ct.jp.nec.com" }, # Takashi Egawa is Authorized Individual in itu-t-jca-sdn +# {"person_id": 107298, "group_id": 2178, "email": "tsbsg11@itu.int" }, # Arshey Odedra is Authorized Individual in itu-tsbsg-11 +# {"person_id": 107300, "group_id": 77, "email": "tatiana.kurakova@itu.int" }, # Tatiana Kurakova is Authorized Individual in itu-t-sg-11 +# {"person_id": 112573, "group_id": 77, "email": "stefano.polidori@itu.int" }, # Stefano Polidori is Authorized Individual in itu-t-sg-11 +# {"person_id": 115401, "group_id": 84, "email": "spennock@rim.com" }, # Scott Pennock is Authorized Individual in itu-t-sg-12 +# {"person_id": 114255, "group_id": 84, "email": "hiroshi.ota@itu.int" }, # Hiroshi Ota is Authorized Individual in itu-t-sg-12 +# {"person_id": 113032, "group_id": 84, "email": "catherine.quinquis@orange.com" }, # Catherine Quinquis is Authorized Individual in itu-t-sg-12 +# {"person_id": 113031, "group_id": 84, "email": "gunilla.berndtsson@ericsson.com" }, # Gunilla Berndtsson is Authorized Individual in itu-t-sg-12 +# {"person_id": 113672, "group_id": 84, "email": "sarah.scott@itu.int" }, # Sarah Scott is Authorized Individual in itu-t-sg-12 +# {"person_id": 122459, "group_id": 81, "email": "chan@etri.re.kr" }, # Kangchan Lee is Authorized Individual in itu-t-sg-13 +# {"person_id": 107300, "group_id": 81, "email": "tatiana.kurakova@itu.int" }, # Tatiana Kurakova is Authorized Individual in itu-t-sg-13 +# {"person_id": 109145, "group_id": 62, "email": "lihan@chinamobile.com" }, # Han Li is Authorized Individual in itu-t-sg-15 +# {"person_id": 115875, "group_id": 62, "email": "mark.jones@xtera.com" }, # Mark Jones is Authorized Individual in itu-t-sg-15 +# {"person_id": 115846, "group_id": 62, "email": "peter.stassar@huawei.com" }, # Peter Stassar is Authorized Individual in itu-t-sg-15 +# {"person_id": 123452, "group_id": 62, "email": "sshew@ciena.com" }, # Stephen Shew is Authorized Individual in itu-t-sg-15 +# {"person_id": 109312, "group_id": 62, "email": "huubatwork@gmail.com" }, # Huub van Helvoort is Authorized Individual in itu-t-sg-15 +# {"person_id": 115874, "group_id": 62, "email": "tom.huber@tellabs.com" }, # Tom Huber is Authorized Individual in itu-t-sg-15 +# {"person_id": 110799, "group_id": 62, "email": "koike.yoshinori@lab.ntt.co.jp" }, # Yoshinori Koike is Authorized Individual in itu-t-sg-15 +# {"person_id": 110831, "group_id": 62, "email": "kam.lam@nokia.com" }, # Hing-Kam Lam is Authorized Individual in itu-t-sg-15 +# {"person_id": 114255, "group_id": 62, "email": "hiroshi.ota@itu.int" }, # Hiroshi Ota is Authorized Individual in itu-t-sg-15 +# {"person_id": 115874, "group_id": 62, "email": "tom.huber@coriant.com" }, # Tom Huber is Authorized Individual in itu-t-sg-15 +# {"person_id": 123014, "group_id": 62, "email": "jessy.rouyer@nokia.com" }, # Jessy Rouyer is Authorized Individual in itu-t-sg-15 +# {"person_id": 111160, "group_id": 62, "email": "ryoo@etri.re.kr" }, # Jeong-dong Ryoo is Authorized Individual in itu-t-sg-15 +# {"person_id": 107296, "group_id": 62, "email": "greg.jones@itu.int" }, # Greg Jones is Authorized Individual in itu-t-sg-15 +# {"person_id": 118539, "group_id": 72, "email": "rosa.angelesleondev@itu.int" }, # Rosa De Vivero is Authorized Individual in itu-t-sg-16 +# {"person_id": 123169, "group_id": 72, "email": "garysull@microsoft.com" }, # Gary Sullivan is Authorized Individual in itu-t-sg-16 +# {"person_id": 107746, "group_id": 72, "email": "hiwasaki.yusuke@lab.ntt.co.jp" }, # Yusuke Hiwasaki is Authorized Individual in itu-t-sg-16 +# {"person_id": 108160, "group_id": 1987, "email": "Christian.Groves@nteczone.com" }, # Christian Groves is Authorized Individual in itu-t-sg-16-q3 +# {"person_id": 118539, "group_id": 1987, "email": "rosa.angelesleondev@itu.int" }, # Rosa De Vivero is Authorized Individual in itu-t-sg-16-q3 +# {"person_id": 124354, "group_id": 76, "email": "jhbaek@kisa.or.kr" }, # Jonghyun Baek is Authorized Individual in itu-t-sg-17 +# {"person_id": 12898, "group_id": 1937, "email": "youki-k@is.aist-nara.ac.jp" }, # Youki Kadobayashi is Authorized Individual in itu-t-sg-17-q4 +# {"person_id": 113593, "group_id": 79, "email": "maite.comasbarnes@itu.int" }, # Maite Barnes is Authorized Individual in itu-t-sg-3 +# {"person_id": 122983, "group_id": 2000, "email": "cristina.bueti@itu.int" }, # Cristina Bueti is Authorized Individual in itu-t-sg-5 +# {"person_id": 112573, "group_id": 2072, "email": "stefano.polidori@itu.int" }, # Stefano Polidori is Authorized Individual in itu-t-sg-9 +# {"person_id": 113101, "group_id": 82, "email": "steve.trowbridge@alcatel-lucent.com" }, # Stephen Trowbridge is Authorized Individual in itu-t-tsag +# {"person_id": 20783, "group_id": 82, "email": "reinhard.scholl@itu.int" }, # Reinhard Scholl is Authorized Individual in itu-t-tsag +# {"person_id": 107300, "group_id": 1846, "email": "tatiana.kurakova@itu.int" }, # Tatiana Kurakova is Authorized Individual in itu-t-wp-5-13 +# {"person_id": 112107, "group_id": 69, "email": "michael.oreirdan@maawg.org" }, # Michael O'Reirdan is Authorized Individual in maawg +# {"person_id": 121870, "group_id": 75, "email": "liaisons@mef.net" }, # Liaison Mef is Authorized Individual in mef +# {"person_id": 112510, "group_id": 75, "email": "nan@mef.net" }, # Nan Chen is Authorized Individual in mef +# {"person_id": 124306, "group_id": 75, "email": "jason.wolfe@bell.ca" }, # WOLFE Jason is Authorized Individual in mef +# {"person_id": 114454, "group_id": 75, "email": "mike.bencheck@siamasystems.com" }, # Mike Bencheck is Authorized Individual in mef +# {"person_id": 115327, "group_id": 1888, "email": "klaus.moschner@ngmn.org" }, # Klaus Moschner is Authorized Individual in ngmn +# {"person_id": 123305, "group_id": 1888, "email": "office@ngmn.org" }, # Office Ngmn is Authorized Individual in ngmn +# {"person_id": 115160, "group_id": 1888, "email": "jminlee@sk.com" }, # Jongmin Lee is Authorized Individual in ngmn +# {"person_id": 117424, "group_id": 1936, "email": "patrick.gallagher@nist.gov" }, # Patrick Gallagher is Authorized Individual in nist +# {"person_id": 117431, "group_id": 1941, "email": "chet.ensign@xn--oasis-open-vt6e.org" }, # Chet Ensign is Authorized Individual in oasis +# {"person_id": 120913, "group_id": 2142, "email": "james.walker@tatacommunications.com" }, # James Walker is Authorized Individual in occ +# {"person_id": 6699, "group_id": 2142, "email": "dromasca@gmail.com" }, # Dan Romascanu is Authorized Individual in occ +# {"person_id": 118403, "group_id": 2142, "email": "richard.schell@verizon.com" }, # Rick Schell is Authorized Individual in occ +# {"person_id": 109676, "group_id": 83, "email": "Jonathan.Sadler@tellabs.com" }, # Jonathan Sadler is Authorized Individual in oif +# {"person_id": 122843, "group_id": 2122, "email": "tzhang@omaorg.org" }, # Tiffany Zhang is Authorized Individual in oma +# {"person_id": 116967, "group_id": 1947, "email": "JMudge@omaorg.org" }, # John Mudge is Authorized Individual in oma-architecture-wg +# {"person_id": 117423, "group_id": 1935, "email": "soley@omg.org" }, # Richard Soley is Authorized Individual in omg +# {"person_id": 110831, "group_id": 1858, "email": "kam.lam@nokia.com" }, # Hing-Kam Lam is Authorized Individual in onf +# {"person_id": 113674, "group_id": 1858, "email": "dan.pitt@opennetworking.org" }, # Dan Pitt is Authorized Individual in onf +# {"person_id": 118348, "group_id": 1984, "email": "dave.hood@ericsson.com" }, # Dave Hood is Authorized Individual in onf-arch-wg +# {"person_id": 116967, "group_id": 60, "email": "JMudge@omaorg.org" }, # John Mudge is Authorized Individual in open-mobile-alliance +# {"person_id": 112613, "group_id": 60, "email": "jerry.shih@att.com" }, # Jerry Shih is Authorized Individual in open-mobile-alliance +# {"person_id": 113067, "group_id": 60, "email": "laurent.goix@econocom.com" }, # Laurent Goix is Authorized Individual in open-mobile-alliance +# {"person_id": 112772, "group_id": 60, "email": "zhiyuan.hu@alcatel-sbell.com.cn" }, # Hu Zhiyuan is Authorized Individual in open-mobile-alliance +# {"person_id": 113064, "group_id": 60, "email": "thierry.berisot@telekom.de" }, # Thierry Berisot is Authorized Individual in open-mobile-alliance +# {"person_id": 124276, "group_id": 2212, "email": "jmisener@qti.qualcomm.com" }, # Jim Misener is Authorized Individual in sae-cell-v2x +# {"person_id": 124278, "group_id": 2212, "email": "Keith.Wilson@sae.org" }, # Keith Wilson is Authorized Individual in sae-cell-v2x +# {"person_id": 124277, "group_id": 2212, "email": "Elizabeth.Perry@sae.org" }, # Elizabeth Perry is Authorized Individual in sae-cell-v2x +# {"person_id": 117430, "group_id": 1940, "email": "admin@trustedcomputinggroup.org" }, # Lindsay Adamson is Authorized Individual in tcg +# {"person_id": 117422, "group_id": 1934, "email": "j.hietala@opengroup.org" }, # Jim Hietala is Authorized Individual in the-open-group +# {"person_id": 112104, "group_id": 53, "email": "rick@unicode.org" }, # Rick McGowan is Authorized Individual in unicode +# {"person_id": 112103, "group_id": 54, "email": "plh@w3.org" }, # Philippe Le Hégaret is Authorized Individual in w3c +# {"person_id": 120261, "group_id": 54, "email": "wendy@seltzer.org" }, # Wendy Seltzer is Authorized Individual in w3c +# {"person_id": 118020, "group_id": 1955, "email": "tiago@wballiance.com" }, # Tiago Rodrigues is Authorized Individual in wba +# {"person_id": 125489, "group_id": 1955, "email": "bruno@wballiance.com" }, # Bruno Tomas is Authorized Individual in wba +# {"person_id": 109129, "group_id": 70, "email": "smccammon@amsl.com" }, # Stephanie McCammon is Authorized Individual in zigbee-alliance +# ]} diff --git a/ietf/group/migrations/0006_remove_liason_contacts.py b/ietf/group/migrations/0006_remove_liason_contacts.py new file mode 100644 index 0000000000..13afd1a53e --- /dev/null +++ b/ietf/group/migrations/0006_remove_liason_contacts.py @@ -0,0 +1,270 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from collections import defaultdict + +from django.db import migrations + +from ietf.person.name import plain_name + + +def get_plain_name(person): + return person.plain or plain_name(person.name) + + +def forward(apps, schema_editor): + """Removes liaison_contact and liaison_cc_contact roles from all groups + + The IAB has decided to remove the liaison_contact and liaison_cc_contact + role concept from the datatracker as the roles are not well understood + and have not been being maintained. + """ + Role = apps.get_model("group", "Role") + GroupEvent = apps.get_model("group", "GroupEvent") + for role_name in ["liaison_contact", "liaison_cc_contact"]: + groups = defaultdict(list) + role_qs = Role.objects.filter(name_id=role_name) + for role in role_qs: + groups[role.group].append(role) + for group in groups: + desc = f"Removed {role_name}: {', '.join([get_plain_name(role.person) for role in groups[group]])}" + GroupEvent.objects.create( + group=group, + by_id=1, # (System) + desc=desc, + ) + role_qs.delete() + + +def reverse(apps, schema_editor): + """Intentionally does nothing""" + pass + + +class Migration(migrations.Migration): + dependencies = [ + ("group", "0005_remove_sdo_authorized_individuals"), + ] + + operations = [ + migrations.RunPython(forward, reverse), + ] + + +# At the time this migration was created, it would remove these objects +# {"liaison_contacts":[ +# { "role_name": "liaison_contact", "person_id": 107943, "group_id": 56, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Liaison Contact in 3gpp +# { "role_name": "liaison_contact", "person_id": 107737, "group_id": 56, "email": "lionel.morand@orange.com" }, # Lionel Morand is Liaison Contact in 3gpp +# { "role_name": "liaison_contact", "person_id": 127959, "group_id": 57, "email": "mahendra@qualcomm.com" }, # Mahendran Ac is Liaison Contact in 3gpp2 +# { "role_name": "liaison_contact", "person_id": 111440, "group_id": 2026, "email": "georg.mayer.huawei@gmx.com" }, # Georg Mayer is Liaison Contact in 3gpp-tsgct +# { "role_name": "liaison_contact", "person_id": 107943, "group_id": 2026, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Liaison Contact in 3gpp-tsgct +# { "role_name": "liaison_contact", "person_id": 107943, "group_id": 2027, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Liaison Contact in 3gpp-tsgct-ct1 +# { "role_name": "liaison_contact", "person_id": 107737, "group_id": 2027, "email": "lionel.morand@orange.com" }, # Lionel Morand is Liaison Contact in 3gpp-tsgct-ct1 +# { "role_name": "liaison_contact", "person_id": 107737, "group_id": 2410, "email": "lionel.morand@orange.com" }, # Lionel Morand is Liaison Contact in 3gpp-tsgct-ct3 +# { "role_name": "liaison_contact", "person_id": 107943, "group_id": 2410, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Liaison Contact in 3gpp-tsgct-ct3 +# { "role_name": "liaison_contact", "person_id": 107737, "group_id": 2028, "email": "lionel.morand@orange.com" }, # Lionel Morand is Liaison Contact in 3gpp-tsgct-ct4 +# { "role_name": "liaison_contact", "person_id": 107943, "group_id": 2028, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Liaison Contact in 3gpp-tsgct-ct4 +# { "role_name": "liaison_contact", "person_id": 107943, "group_id": 2029, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Liaison Contact in 3gpp-tsgran +# { "role_name": "liaison_contact", "person_id": 111440, "group_id": 2029, "email": "georg.mayer.huawei@gmx.com" }, # Georg Mayer is Liaison Contact in 3gpp-tsgran +# { "role_name": "liaison_contact", "person_id": 107737, "group_id": 2030, "email": "lionel.morand@orange.com" }, # Lionel Morand is Liaison Contact in 3gpp-tsgran-ran2 +# { "role_name": "liaison_contact", "person_id": 107943, "group_id": 2030, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Liaison Contact in 3gpp-tsgran-ran2 +# { "role_name": "liaison_contact", "person_id": 107943, "group_id": 2023, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Liaison Contact in 3gpp-tsgsa +# { "role_name": "liaison_contact", "person_id": 111440, "group_id": 2023, "email": "georg.mayer.huawei@gmx.com" }, # Georg Mayer is Liaison Contact in 3gpp-tsgsa +# { "role_name": "liaison_contact", "person_id": 107943, "group_id": 2024, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Liaison Contact in 3gpp-tsgsa-sa2 +# { "role_name": "liaison_contact", "person_id": 107737, "group_id": 2024, "email": "lionel.morand@orange.com" }, # Lionel Morand is Liaison Contact in 3gpp-tsgsa-sa2 +# { "role_name": "liaison_contact", "person_id": 107943, "group_id": 2025, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Liaison Contact in 3gpp-tsgsa-sa3 +# { "role_name": "liaison_contact", "person_id": 107737, "group_id": 2025, "email": "lionel.morand@orange.com" }, # Lionel Morand is Liaison Contact in 3gpp-tsgsa-sa3 +# { "role_name": "liaison_contact", "person_id": 107737, "group_id": 1902, "email": "lionel.morand@orange.com" }, # Lionel Morand is Liaison Contact in 3gpp-tsgsa-sa4 +# { "role_name": "liaison_contact", "person_id": 107943, "group_id": 1902, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Liaison Contact in 3gpp-tsgsa-sa4 +# { "role_name": "liaison_contact", "person_id": 107943, "group_id": 2031, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Liaison Contact in 3gpp-tsgt-wg2 +# { "role_name": "liaison_contact", "person_id": 107737, "group_id": 2031, "email": "lionel.morand@orange.com" }, # Lionel Morand is Liaison Contact in 3gpp-tsgt-wg2 +# { "role_name": "liaison_contact", "person_id": 106345, "group_id": 1396, "email": "Menachem.Dodge@ecitele.com" }, # Menachem Dodge is Liaison Contact in adslmib +# { "role_name": "liaison_contact", "person_id": 108054, "group_id": 1956, "email": "shengjiang@bupt.edu.cn" }, # Sheng Jiang is Liaison Contact in anima +# { "role_name": "liaison_contact", "person_id": 11834, "group_id": 1956, "email": "tte@cs.fau.de" }, # Toerless Eckert is Liaison Contact in anima +# { "role_name": "liaison_contact", "person_id": 21684, "group_id": 1805, "email": "barryleiba@computer.org" }, # Barry Leiba is Liaison Contact in appsawg +# { "role_name": "liaison_contact", "person_id": 102154, "group_id": 1805, "email": "alexey.melnikov@isode.com" }, # Alexey Melnikov is Liaison Contact in appsawg +# { "role_name": "liaison_contact", "person_id": 107279, "group_id": 1805, "email": "yaojk@cnnic.cn" }, # Jiankang Yao is Liaison Contact in appsawg +# { "role_name": "liaison_contact", "person_id": 100754, "group_id": 941, "email": "tom.taylor@rogers.com" }, # Tom Taylor is Liaison Contact in avt +# { "role_name": "liaison_contact", "person_id": 105873, "group_id": 941, "email": "ron.even.tlv@gmail.com" }, # Roni Even is Liaison Contact in avt +# { "role_name": "liaison_contact", "person_id": 105097, "group_id": 1813, "email": "keith.drage@alcatel-lucent.com" }, # Keith Drage is Liaison Contact in avtext +# { "role_name": "liaison_contact", "person_id": 101923, "group_id": 1813, "email": "jonathan@vidyo.com" }, # Jonathan Lennox is Liaison Contact in avtext +# { "role_name": "liaison_contact", "person_id": 108279, "group_id": 1960, "email": "martin.vigoureux@alcatel-lucent.com" }, # Martin Vigoureux is Liaison Contact in bess +# { "role_name": "liaison_contact", "person_id": 109666, "group_id": 66, "email": "g.white@cablelabs.com" }, # Greg White is Liaison Contact in cablelabs +# { "role_name": "liaison_contact", "person_id": 117421, "group_id": 1933, "email": "chairman@dmtf.org" }, # Winston Bumpus is Liaison Contact in dmtf +# { "role_name": "liaison_contact", "person_id": 127961, "group_id": 1739, "email": "statements@ietf.org" }, # statements@ietf.org is Liaison Contact in drinks +# { "role_name": "liaison_contact", "person_id": 109505, "group_id": 1787, "email": "bernie@ietf.hoeneisen.ch" }, # Bernie Hoeneisen is Liaison Contact in e2md +# { "role_name": "liaison_contact", "person_id": 109059, "group_id": 1787, "email": "ray.bellis@nominet.org.uk" }, # Ray Bellis is Liaison Contact in e2md +# { "role_name": "liaison_contact", "person_id": 116529, "group_id": 1919, "email": "istvan@ecma-interational.org" }, # Istvan Sebestyen is Liaison Contact in ecma-tc39 +# { "role_name": "liaison_contact", "person_id": 127964, "group_id": 1919, "email": "johnneumann.openstrat@gmail.com" }, # John Neuman is Liaison Contact in ecma-tc39 +# { "role_name": "liaison_contact", "person_id": 106012, "group_id": 1643, "email": "marc.linsner@cisco.com" }, # Marc Linsner is Liaison Contact in ecrit +# { "role_name": "liaison_contact", "person_id": 107084, "group_id": 1643, "email": "rmarshall@telecomsys.com" }, # Roger Marshall is Liaison Contact in ecrit +# { "role_name": "liaison_contact", "person_id": 116363, "group_id": 1915, "email": "e2nasupport@etsi.org" }, # Sonia Compans is Liaison Contact in etsi-e2na +# { "role_name": "liaison_contact", "person_id": 126473, "group_id": 2261, "email": "isgsupport@etsi.org" }, # Sonia Compan is Liaison Contact in etsi-isg-sai +# { "role_name": "liaison_contact", "person_id": 128316, "group_id": 2301, "email": "GSMALiaisons@gsma.com" }, # David Pollington is Liaison Contact in gsma-ztc +# { "role_name": "liaison_contact", "person_id": 3056, "group_id": 1875, "email": "shares@ndzh.com" }, # Susan Hares is Liaison Contact in i2rs +# { "role_name": "liaison_contact", "person_id": 105046, "group_id": 1875, "email": "jhaas@pfrc.org" }, # Jeffrey Haas is Liaison Contact in i2rs +# { "role_name": "liaison_contact", "person_id": 120845, "group_id": 61, "email": "tale@dd.org" }, # David Lawrence is Liaison Contact in icann-board-of-directors +# { "role_name": "liaison_contact", "person_id": 112851, "group_id": 2105, "email": "pthaler@broadcom.com" }, # Patricia Thaler is Liaison Contact in ieee-802 +# { "role_name": "liaison_contact", "person_id": 127968, "group_id": 2105, "email": "p.nikolich@ieee.org" }, # Paul Nikolich is Liaison Contact in ieee-802 +# { "role_name": "liaison_contact", "person_id": 19651, "group_id": 63, "email": "glenn.parsons@ericsson.com" }, # Glenn Parsons is Liaison Contact in ieee-802-1 +# { "role_name": "liaison_contact", "person_id": 123875, "group_id": 63, "email": "JMessenger@advaoptical.com" }, # John Messenger is Liaison Contact in ieee-802-1 +# { "role_name": "liaison_contact", "person_id": 127968, "group_id": 63, "email": "p.nikolich@ieee.org" }, # Paul Nikolich is Liaison Contact in ieee-802-1 +# { "role_name": "liaison_contact", "person_id": 117415, "group_id": 1862, "email": "Adrian.P.Stephens@intel.com" }, # Adrian Stephens is Liaison Contact in ieee-802-11 +# { "role_name": "liaison_contact", "person_id": 106284, "group_id": 1862, "email": "dstanley@agere.com" }, # Dorothy Stanley is Liaison Contact in ieee-802-11 +# { "role_name": "liaison_contact", "person_id": 128345, "group_id": 2302, "email": "liaison@iowngf.org" }, # Forum Iown is Liaison Contact in iown-global-forum +# { "role_name": "liaison_contact", "person_id": 117428, "group_id": 1939, "email": "walter.fumy@bdr.de" }, # Walter Fumy is Liaison Contact in iso-iec-jtc1-sc27 +# { "role_name": "liaison_contact", "person_id": 117429, "group_id": 1939, "email": "krystyna.passia@din.de" }, # Krystyna Passia is Liaison Contact in iso-iec-jtc1-sc27 +# { "role_name": "liaison_contact", "person_id": 151289, "group_id": 50, "email": "koike@itscj.ipsj.or.jp" }, # Mayumi Koike is Liaison Contact in iso-iec-jtc1-sc29 +# { "role_name": "liaison_contact", "person_id": 151289, "group_id": 2110, "email": "koike@itscj.ipsj.or.jp" }, # Mayumi Koike is Liaison Contact in iso-iec-jtc1-sc29-wg1 +# { "role_name": "liaison_contact", "person_id": 114435, "group_id": 74, "email": "watanabe@itscj.ipsj.or.jp" }, # Shinji Watanabe is Liaison Contact in iso-iec-jtc1-sc29-wg11 +# { "role_name": "liaison_contact", "person_id": 112106, "group_id": 49, "email": "jooran@kisi.or.kr" }, # Jooran Lee is Liaison Contact in iso-iec-jtc1-sc6 +# { "role_name": "liaison_contact", "person_id": 113587, "group_id": 49, "email": "dykim@cnu.kr" }, # Chungnam University is Liaison Contact in iso-iec-jtc1-sc6 +# { "role_name": "liaison_contact", "person_id": 117427, "group_id": 1938, "email": "secretariat@jtc1-sc7.org" }, # Witold Suryn is Liaison Contact in iso-iec-jtc1-sc7 +# { "role_name": "liaison_contact", "person_id": 117426, "group_id": 1938, "email": "chair@jtc1-sc7.org" }, # Francois Coallier is Liaison Contact in iso-iec-jtc1-sc7 +# { "role_name": "liaison_contact", "person_id": 127971, "group_id": 68, "email": "sabine.donnardcusse@afnor.org" }, # sabine.donnardcusse@afnor.org is Liaison Contact in isotc46 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2057, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 1890, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-r +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2058, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-r-wp5a +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2059, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-r-wp5d +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2060, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-r-wp8a +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2061, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-r-wp8f +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 51, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2063, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-fg-cloud +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 1860, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-fg-dist +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2064, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-fg-iptv +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2065, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-fg-ngnm +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2062, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-ipv6-group +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 1872, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-jca-cloud +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 1874, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-jca-cop +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2066, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-jca-idm +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 1927, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-jca-sdn +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 65, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-mpls +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 52, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-ngn +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2067, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-ngnmfg +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 77, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-11 +# { "role_name": "liaison_contact", "person_id": 128236, "group_id": 77, "email": "denis.andreev@itu.int" }, # Denis ANDREEV is Liaison Contact in itu-t-sg-11 +# { "role_name": "liaison_contact", "person_id": 107300, "group_id": 77, "email": "tatiana.kurakova@itu.int" }, # Tatiana Kurakova is Liaison Contact in itu-t-sg-11 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2074, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-11-q5 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2075, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-11-wp2 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 84, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-12 +# { "role_name": "liaison_contact", "person_id": 102900, "group_id": 84, "email": "acmorton@att.com" }, # Al Morton is Liaison Contact in itu-t-sg-12 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2076, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-12-q12 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2077, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-12-q17 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2082, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-13-q11 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2078, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-13-q3 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2079, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-13-q5 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2080, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-13-q7 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2081, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-13-q9 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2083, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-13-wp3 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2084, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-13-wp4 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2085, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-13-wp5 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2086, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-14 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 62, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-15 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2087, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-15-q1 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2092, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-15-q10 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2093, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-15-q11 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2094, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-15-q12 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2095, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-15-q14 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2096, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-15-q15 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2088, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-15-q3 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2089, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-15-q4 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2090, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-15-q6 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2091, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-15-q9 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2097, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-15-wp1 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2098, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-15-wp3 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 72, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-16 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2101, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-16-q10 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 1987, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-16-q3 +# { "role_name": "liaison_contact", "person_id": 118539, "group_id": 1987, "email": "rosa.angelesleondev@itu.int" }, # Rosa De Vivero is Liaison Contact in itu-t-sg-16-q3 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2099, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-16-q8 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2100, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-16-q9 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 76, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-17 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2102, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-17-q2 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 1937, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-17-q4 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 1954, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-17-tsb +# { "role_name": "liaison_contact", "person_id": 12898, "group_id": 1954, "email": "youki-k@is.aist-nara.ac.jp" }, # Youki Kadobayashi is Liaison Contact in itu-t-sg-17-tsb +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 78, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-2 +# { "role_name": "liaison_contact", "person_id": 127962, "group_id": 78, "email": "dr.guinena@ntra.gov.eg" }, # dr.guinena@ntra.gov.eg is Liaison Contact in itu-t-sg-2 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2103, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-20 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2073, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-2-q1 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 79, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-3 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2068, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-4 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2000, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-5 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2069, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-6 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2070, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-7 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2071, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-8 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 2072, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-sg-9 +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 82, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-tsag +# { "role_name": "liaison_contact", "person_id": 127957, "group_id": 82, "email": "tsbtsag@itu.int" }, # Bilel Jamoussi is Liaison Contact in itu-t-tsag +# { "role_name": "liaison_cc_contact", "person_id": 127958, "group_id": 1846, "email": "itu-t-liaison@iab.org" }, # itu-t liaison is Liaison CC Contact in itu-t-wp-5-13 +# { "role_name": "liaison_contact", "person_id": 10083, "group_id": 1882, "email": "paul.hoffman@vpnc.org" }, # Paul Hoffman is Liaison Contact in json +# { "role_name": "liaison_contact", "person_id": 111178, "group_id": 1882, "email": "mamille2@cisco.com" }, # Matthew Miller is Liaison Contact in json +# { "role_name": "liaison_contact", "person_id": 106881, "group_id": 1593, "email": "vach.kompella@alcatel.com" }, # Vach Kompella is Liaison Contact in l2vpn +# { "role_name": "liaison_contact", "person_id": 19987, "group_id": 1593, "email": "danny@arbor.net" }, # Danny McPherson is Liaison Contact in l2vpn +# { "role_name": "liaison_contact", "person_id": 2329, "group_id": 1593, "email": "stbryant@cisco.com" }, # Stewart Bryant is Liaison Contact in l2vpn +# { "role_name": "liaison_contact", "person_id": 101552, "group_id": 1593, "email": "Shane.Amante@Level3.com" }, # Shane Amante is Liaison Contact in l2vpn +# { "role_name": "liaison_contact", "person_id": 110305, "group_id": 1877, "email": "jason.weil@twcable.com" }, # Jason Weil is Liaison Contact in lmap +# { "role_name": "liaison_contact", "person_id": 6699, "group_id": 1877, "email": "dromasca@avaya.com" }, # Dan Romascanu is Liaison Contact in lmap +# { "role_name": "liaison_contact", "person_id": 127969, "group_id": 69, "email": "madkins@fb.com" }, # Mike Adkins is Liaison Contact in maawg +# { "role_name": "liaison_contact", "person_id": 127970, "group_id": 69, "email": "technical-chair@mailman.m3aawg.org" }, # technical-chair@mailman.m3aawg.org is Liaison Contact in maawg +# { "role_name": "liaison_contact", "person_id": 112512, "group_id": 75, "email": "rraghu@ciena.com" }, # Raghu Ranganathan is Liaison Contact in mef +# { "role_name": "liaison_contact", "person_id": 119947, "group_id": 1755, "email": "mrw@lilacglade.org" }, # Margaret Cullen is Liaison Contact in mif +# { "role_name": "liaison_contact", "person_id": 109884, "group_id": 1755, "email": "denghui02@hotmail.com" }, # Hui Deng is Liaison Contact in mif +# { "role_name": "liaison_contact", "person_id": 128292, "group_id": 1936, "email": "james.olthoff@nist.gov" }, # James Olthoff is Liaison Contact in nist +# { "role_name": "liaison_contact", "person_id": 104183, "group_id": 1537, "email": "john.loughney@nokia.com" }, # John Loughney is Liaison Contact in nsis +# { "role_name": "liaison_contact", "person_id": 105786, "group_id": 1840, "email": "matthew.bocci@nokia.com" }, # Matthew Bocci is Liaison Contact in nvo3 +# { "role_name": "liaison_contact", "person_id": 112438, "group_id": 1840, "email": "bensons@queuefull.net" }, # Benson Schliesser is Liaison Contact in nvo3 +# { "role_name": "liaison_contact", "person_id": 107943, "group_id": 2296, "email": "3GPPLiaison@etsi.org" }, # Susanna Kooistra is Liaison Contact in o3gpptsgran3 +# { "role_name": "liaison_contact", "person_id": 127966, "group_id": 1941, "email": "chet.ensign@oasis-open.org" }, # chet.ensign@oasis-open.org is Liaison Contact in oasis +# { "role_name": "liaison_contact", "person_id": 117423, "group_id": 1935, "email": "soley@omg.org" }, # Richard Soley is Liaison Contact in omg +# { "role_name": "liaison_contact", "person_id": 127963, "group_id": 1858, "email": "dan.pitt@opennetworkingfoundation.org" }, # dan.pitt@opennetworkingfoundation.org is Liaison Contact in onf +# { "role_name": "liaison_contact", "person_id": 108304, "group_id": 1599, "email": "gunter.van_de_velde@nokia.com" }, # Gunter Van de Velde is Liaison Contact in opsec +# { "role_name": "liaison_contact", "person_id": 111647, "group_id": 1599, "email": "kk@google.com" }, # Chittimaneni Kk is Liaison Contact in opsec +# { "role_name": "liaison_contact", "person_id": 111656, "group_id": 1599, "email": "warren@kumari.net" }, # Warren Kumari is Liaison Contact in opsec +# { "role_name": "liaison_contact", "person_id": 106471, "group_id": 1188, "email": "dbrungard@att.com" }, # Deborah Brungard is Liaison Contact in ospf +# { "role_name": "liaison_contact", "person_id": 104198, "group_id": 1188, "email": "adrian@olddog.co.uk" }, # Adrian Farrel is Liaison Contact in ospf +# { "role_name": "liaison_contact", "person_id": 104816, "group_id": 1188, "email": "akr@cisco.com" }, # Abhay Roy is Liaison Contact in ospf +# { "role_name": "liaison_contact", "person_id": 10784, "group_id": 1188, "email": "acee@redback.com" }, # Acee Lindem is Liaison Contact in ospf +# { "role_name": "liaison_contact", "person_id": 108123, "group_id": 1819, "email": "Gabor.Bajko@nokia.com" }, # Gabor Bajko is Liaison Contact in paws +# { "role_name": "liaison_contact", "person_id": 106987, "group_id": 1819, "email": "br@brianrosen.net" }, # Brian Rosen is Liaison Contact in paws +# { "role_name": "liaison_cc_contact", "person_id": 122823, "group_id": 1630, "email": "ketant.ietf@gmail.com" }, # Ketan Talaulikar is Liaison CC Contact in pce +# { "role_name": "liaison_contact", "person_id": 125031, "group_id": 1630, "email": "andrew.stone@nokia.com" }, # Andrew Stone is Liaison Contact in pce +# { "role_name": "liaison_contact", "person_id": 108213, "group_id": 1630, "email": "julien.meuric@orange.com" }, # Julien Meuric is Liaison Contact in pce +# { "role_name": "liaison_contact", "person_id": 111477, "group_id": 1630, "email": "dd@dhruvdhody.com" }, # Dhruv Dhody is Liaison Contact in pce +# { "role_name": "liaison_contact", "person_id": 112773, "group_id": 1701, "email": "lars.eggert@nokia.com" }, # Lars Eggert is Liaison Contact in pcn +# { "role_name": "liaison_contact", "person_id": 12671, "group_id": 1437, "email": "adamson@itd.nrl.navy.mil" }, # Brian Adamson is Liaison Contact in rmt +# { "role_name": "liaison_contact", "person_id": 100609, "group_id": 1437, "email": "lorenzo@vicisano.net" }, # Lorenzo Vicisano is Liaison Contact in rmt +# { "role_name": "liaison_contact", "person_id": 115213, "group_id": 1730, "email": "maria.ines.robles@ericsson.com" }, # Ines Robles is Liaison Contact in roll +# { "role_name": "liaison_contact", "person_id": 110721, "group_id": 1820, "email": "ted.ietf@gmail.com" }, # Ted Hardie is Liaison Contact in rtcweb +# { "role_name": "liaison_contact", "person_id": 104294, "group_id": 1820, "email": "magnus.westerlund@ericsson.com" }, # Magnus Westerlund is Liaison Contact in rtcweb +# { "role_name": "liaison_contact", "person_id": 105791, "group_id": 1820, "email": "fluffy@iii.ca" }, # Cullen Jennings is Liaison Contact in rtcweb +# { "role_name": "liaison_contact", "person_id": 105906, "group_id": 1910, "email": "james.n.guichard@futurewei.com" }, # Jim Guichard is Liaison Contact in sfc +# { "role_name": "liaison_contact", "person_id": 3862, "group_id": 1910, "email": "jmh@joelhalpern.com" }, # Joel Halpern is Liaison Contact in sfc +# { "role_name": "liaison_contact", "person_id": 127960, "group_id": 1462, "email": "sipcore@ietf.org" }, # sipcore@ietf.org is Liaison Contact in sip +# { "role_name": "liaison_contact", "person_id": 103769, "group_id": 1762, "email": "adam@nostrum.com" }, # Adam Roach is Liaison Contact in sipcore +# { "role_name": "liaison_contact", "person_id": 108554, "group_id": 1762, "email": "pkyzivat@alum.mit.edu" }, # Paul Kyzivat is Liaison Contact in sipcore +# { "role_name": "liaison_contact", "person_id": 103539, "group_id": 1542, "email": "gonzalo.camarillo@ericsson.com" }, # Gonzalo Camarillo is Liaison Contact in sipping +# { "role_name": "liaison_contact", "person_id": 103612, "group_id": 1542, "email": "jf.mule@cablelabs.com" }, # Jean-Francois Mule is Liaison Contact in sipping +# { "role_name": "liaison_contact", "person_id": 3862, "group_id": 1905, "email": "jmh@joelhalpern.com" }, # Joel Halpern is Liaison Contact in spring +# { "role_name": "liaison_contact", "person_id": 109802, "group_id": 1905, "email": "aretana.ietf@gmail.com" }, # Alvaro Retana is Liaison Contact in spring +# { "role_name": "liaison_contact", "person_id": 107172, "group_id": 1905, "email": "bruno.decraene@orange.com" }, # Bruno Decraene is Liaison Contact in spring +# { "role_name": "liaison_contact", "person_id": 5376, "group_id": 1899, "email": "housley@vigilsec.com" }, # Russ Housley is Liaison Contact in stir +# { "role_name": "liaison_contact", "person_id": 103961, "group_id": 1899, "email": "rjsparks@nostrum.com" }, # Robert Sparks is Liaison Contact in stir +# { "role_name": "liaison_contact", "person_id": 117430, "group_id": 1940, "email": "admin@trustedcomputinggroup.org" }, # Lindsay Adamson is Liaison Contact in tcg +# { "role_name": "liaison_contact", "person_id": 110932, "group_id": 1985, "email": "oscar.gonzalezdedios@telefonica.com" }, # Oscar de Dios is Liaison Contact in teas +# { "role_name": "liaison_contact", "person_id": 10064, "group_id": 1985, "email": "lberger@labn.net" }, # Lou Berger is Liaison Contact in teas +# { "role_name": "liaison_contact", "person_id": 114351, "group_id": 1985, "email": "vbeeram@juniper.net" }, # Vishnu Beeram is Liaison Contact in teas +# { "role_name": "liaison_contact", "person_id": 117422, "group_id": 1934, "email": "j.hietala@opengroup.org" }, # Jim Hietala is Liaison Contact in the-open-group +# { "role_name": "liaison_contact", "person_id": 106414, "group_id": 1709, "email": "yaakovjstein@gmail.com" }, # Yaakov Stein is Liaison Contact in tictoc +# { "role_name": "liaison_contact", "person_id": 4857, "group_id": 1709, "email": "kodonog@pobox.com" }, # Karen O'Donoghue is Liaison Contact in tictoc +# { "role_name": "liaison_contact", "person_id": 144713, "group_id": 2420, "email": "liaisons@tmforum.org" }, # liaisons@tmforum.org is Liaison Contact in tmforum +# { "role_name": "liaison_contact", "person_id": 112773, "group_id": 1324, "email": "lars@eggert.org" }, # Lars Eggert is Liaison Contact in tsv +# { "role_name": "liaison_contact", "person_id": 112104, "group_id": 53, "email": "rick@unicode.org" }, # Rick McGowan is Liaison Contact in unicode +# { "role_name": "liaison_contact", "person_id": 105907, "group_id": 1864, "email": "stpeter@stpeter.im" }, # Peter Saint-Andre is Liaison Contact in videocodec +# { "role_name": "liaison_contact", "person_id": 120261, "group_id": 54, "email": "wseltzer@w3.org" }, # Wendy Seltzer is Liaison Contact in w3c +# { "role_name": "liaison_contact", "person_id": 112103, "group_id": 54, "email": "plh@w3.org" }, # Philippe Le Hégaret is Liaison Contact in w3c +# { "role_name": "liaison_contact", "person_id": 107520, "group_id": 1957, "email": "shida@ntt-at.com" }, # Shida Schubert is Liaison Contact in webpush +# { "role_name": "liaison_contact", "person_id": 110049, "group_id": 1957, "email": "jhildebr@cisco.com" }, # Joe Hildebrand is Liaison Contact in webpush +# { "role_name": "liaison_contact", "person_id": 103769, "group_id": 1601, "email": "adam@nostrum.com" }, # Adam Roach is Liaison Contact in xcon +# { "role_name": "liaison_contact", "person_id": 107520, "group_id": 1815, "email": "shida@ntt-at.com" }, # Shida Schubert is Liaison Contact in xrblock +# { "role_name": "liaison_contact", "person_id": 6699, "group_id": 1815, "email": "dromasca@avaya.com" }, # Dan Romascanu is Liaison Contact in xrblock +# ]} diff --git a/ietf/group/migrations/0007_used_roles.py b/ietf/group/migrations/0007_used_roles.py new file mode 100644 index 0000000000..0dfa79fa03 --- /dev/null +++ b/ietf/group/migrations/0007_used_roles.py @@ -0,0 +1,49 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations + + +def forward(apps, schema_editor): + Group = apps.get_model("group", "Group") + GroupFeatures = apps.get_model("group", "GroupFeatures") + iab = Group.objects.get(acronym="iab") + iab.used_roles = [ + "chair", + "delegate", + "exofficio", + "liaison", + "liaison_coordinator", + "member", + ] + iab.save() + GroupFeatures.objects.filter(type_id="ietf").update( + default_used_roles=[ + "ad", + "member", + "comdir", + "delegate", + "execdir", + "recman", + "secr", + "chair", + ] + ) + + +def reverse(apps, schema_editor): + Group = apps.get_model("group", "Group") + iab = Group.objects.get(acronym="iab") + iab.used_roles = [] + iab.save() + # Intentionally not putting trac-* back into grouptype ietf default_used_roles + + +class Migration(migrations.Migration): + dependencies = [ + ("group", "0006_remove_liason_contacts"), + ("name", "0018_alter_rolenames"), + ] + + operations = [ + migrations.RunPython(forward, reverse), + ] diff --git a/ietf/group/migrations/0008_alter_group_used_roles_and_more.py b/ietf/group/migrations/0008_alter_group_used_roles_and_more.py new file mode 100644 index 0000000000..28f345df00 --- /dev/null +++ b/ietf/group/migrations/0008_alter_group_used_roles_and_more.py @@ -0,0 +1,107 @@ +# Generated by Django 4.2.23 on 2025-08-15 16:46 + +from django.db import migrations, models +import ietf.group.models +import ietf.name.models +import ietf.utils.db +import ietf.utils.validators + + +class Migration(migrations.Migration): + + dependencies = [ + ("group", "0007_used_roles"), + ] + + operations = [ + migrations.AlterField( + model_name="group", + name="used_roles", + field=models.JSONField( + blank=True, + default=list, + help_text="Leave an empty list to get the group_type's default used roles", + max_length=256, + ), + ), + migrations.AlterField( + model_name="groupfeatures", + name="admin_roles", + field=ietf.utils.db.EmptyAwareJSONField( + default=ietf.group.models.default_admin_roles, max_length=64 + ), + ), + migrations.AlterField( + model_name="groupfeatures", + name="default_used_roles", + field=ietf.utils.db.EmptyAwareJSONField(default=list, max_length=256), + ), + migrations.AlterField( + model_name="groupfeatures", + name="docman_roles", + field=ietf.utils.db.EmptyAwareJSONField( + default=ietf.group.models.default_docman_roles, max_length=128 + ), + ), + migrations.AlterField( + model_name="groupfeatures", + name="groupman_authroles", + field=ietf.utils.db.EmptyAwareJSONField( + default=ietf.group.models.default_groupman_authroles, max_length=128 + ), + ), + migrations.AlterField( + model_name="groupfeatures", + name="groupman_roles", + field=ietf.utils.db.EmptyAwareJSONField( + default=ietf.group.models.default_groupman_roles, max_length=128 + ), + ), + migrations.AlterField( + model_name="groupfeatures", + name="material_types", + field=ietf.utils.db.EmptyAwareJSONField( + default=ietf.group.models.default_material_types, max_length=64 + ), + ), + migrations.AlterField( + model_name="groupfeatures", + name="matman_roles", + field=ietf.utils.db.EmptyAwareJSONField( + default=ietf.group.models.default_matman_roles, max_length=128 + ), + ), + migrations.AlterField( + model_name="groupfeatures", + name="role_order", + field=ietf.utils.db.EmptyAwareJSONField( + default=ietf.group.models.default_role_order, + help_text="The order in which roles are shown, for instance on photo pages. Enter valid JSON.", + max_length=128, + ), + ), + migrations.AlterField( + model_name="groupfeatures", + name="session_purposes", + field=ietf.utils.db.EmptyAwareJSONField( + default=list, + help_text="Allowed session purposes for this group type", + max_length=256, + validators=[ + ietf.utils.validators.JSONForeignKeyListValidator( + ietf.name.models.SessionPurposeName + ) + ], + ), + ), + migrations.AlterField( + model_name="grouphistory", + name="used_roles", + field=models.JSONField( + blank=True, + default=list, + help_text="Leave an empty list to get the group_type's default used roles", + max_length=256, + ), + ), + ] diff --git a/ietf/group/models.py b/ietf/group/models.py index 52549e8cc1..a7e3c6616e 100644 --- a/ietf/group/models.py +++ b/ietf/group/models.py @@ -3,7 +3,6 @@ import email.utils -import jsonfield import os import re @@ -21,7 +20,7 @@ AgendaTypeName, AgendaFilterTypeName, ExtResourceName, SessionPurposeName, AppealArtifactTypeName ) from ietf.person.models import Email, Person -from ietf.utils.db import IETFJSONField +from ietf.utils.db import EmptyAwareJSONField from ietf.utils.mail import formataddr, send_mail_text from ietf.utils import log from ietf.utils.models import ForeignKey, OneToOneField @@ -46,7 +45,7 @@ class GroupInfo(models.Model): unused_states = models.ManyToManyField('doc.State', help_text="Document states that have been disabled for the group.", blank=True) unused_tags = models.ManyToManyField(DocTagName, help_text="Document tags that have been disabled for the group.", blank=True) - used_roles = jsonfield.JSONField(max_length=256, blank=True, default=[], help_text="Leave an empty list to get the group_type's default used roles") + used_roles = models.JSONField(max_length=256, blank=True, default=list, help_text="Leave an empty list to get the group_type's default used roles") uses_milestone_dates = models.BooleanField(default=True) @@ -112,6 +111,9 @@ def active_wgs(self): def closed_wgs(self): return self.wgs().exclude(state__in=Group.ACTIVE_STATE_IDS) + def areas(self): + return self.get_queryset().filter(type="area") + def with_meetings(self): return self.get_queryset().filter(type__features__has_meetings=True) @@ -235,6 +237,36 @@ def chat_archive_url(self): ) +# JSONFields need callable defaults that work with migrations to avoid sharing +# data structures between instances. These helpers provide that. +def default_material_types(): + return ["slides"] + + +def default_admin_roles(): + return ["chair"] + + +def default_docman_roles(): + return ["ad", "chair", "delegate", "secr"] + + +def default_groupman_roles(): + return ["ad", "chair"] + + +def default_groupman_authroles(): + return ["Secretariat"] + + +def default_matman_roles(): + return ["ad", "chair", "delegate", "secr"] + + +def default_role_order(): + return ["chair", "secr", "member"] + + class GroupFeatures(models.Model): type = OneToOneField(GroupTypeName, primary_key=True, null=False, related_name='features') #history = HistoricalRecords() @@ -268,16 +300,16 @@ class GroupFeatures(models.Model): agenda_type = models.ForeignKey(AgendaTypeName, null=True, default="ietf", on_delete=CASCADE) about_page = models.CharField(max_length=64, blank=False, default="ietf.group.views.group_about" ) default_tab = models.CharField(max_length=64, blank=False, default="ietf.group.views.group_about" ) - material_types = IETFJSONField(max_length=64, accepted_empty_values=[[], {}], blank=False, default=["slides"]) - default_used_roles = IETFJSONField(max_length=256, accepted_empty_values=[[], {}], blank=False, default=[]) - admin_roles = IETFJSONField(max_length=64, accepted_empty_values=[[], {}], blank=False, default=["chair"]) # Trac Admin - docman_roles = IETFJSONField(max_length=128, accepted_empty_values=[[], {}], blank=False, default=["ad","chair","delegate","secr"]) - groupman_roles = IETFJSONField(max_length=128, accepted_empty_values=[[], {}], blank=False, default=["ad","chair",]) - groupman_authroles = IETFJSONField(max_length=128, accepted_empty_values=[[], {}], blank=False, default=["Secretariat",]) - matman_roles = IETFJSONField(max_length=128, accepted_empty_values=[[], {}], blank=False, default=["ad","chair","delegate","secr"]) - role_order = IETFJSONField(max_length=128, accepted_empty_values=[[], {}], blank=False, default=["chair","secr","member"], - help_text="The order in which roles are shown, for instance on photo pages. Enter valid JSON.") - session_purposes = IETFJSONField(max_length=256, accepted_empty_values=[[], {}], blank=False, default=[], + material_types = EmptyAwareJSONField(max_length=64, accepted_empty_values=[[], {}], blank=False, default=default_material_types) + default_used_roles = EmptyAwareJSONField(max_length=256, accepted_empty_values=[[], {}], blank=False, default=list) + admin_roles = EmptyAwareJSONField(max_length=64, accepted_empty_values=[[], {}], blank=False, default=default_admin_roles) # Trac Admin + docman_roles = EmptyAwareJSONField(max_length=128, accepted_empty_values=[[], {}], blank=False, default=default_docman_roles) + groupman_roles = EmptyAwareJSONField(max_length=128, accepted_empty_values=[[], {}], blank=False, default=default_groupman_roles) + groupman_authroles = EmptyAwareJSONField(max_length=128, accepted_empty_values=[[], {}], blank=False, default=default_groupman_authroles) + matman_roles = EmptyAwareJSONField(max_length=128, accepted_empty_values=[[], {}], blank=False, default=default_matman_roles) + role_order = EmptyAwareJSONField(max_length=128, accepted_empty_values=[[], {}], blank=False, default=default_role_order, + help_text="The order in which roles are shown, for instance on photo pages. Enter valid JSON.") + session_purposes = EmptyAwareJSONField(max_length=256, accepted_empty_values=[[], {}], blank=False, default=list, help_text="Allowed session purposes for this group type", validators=[JSONForeignKeyListValidator(SessionPurposeName)]) @@ -462,6 +494,8 @@ def notify_rfceditor_of_group_name_change(sender, instance=None, **kwargs): current = Group.objects.get(pk=instance.pk) except Group.DoesNotExist: return + if current.type_id == "sdo": + return addr = settings.RFC_EDITOR_GROUP_NOTIFICATION_EMAIL if addr and instance.name != current.name: msg = """ diff --git a/ietf/group/serializers.py b/ietf/group/serializers.py new file mode 100644 index 0000000000..e789ba46bf --- /dev/null +++ b/ietf/group/serializers.py @@ -0,0 +1,50 @@ +# Copyright The IETF Trust 2024-2026, All Rights Reserved +"""django-rest-framework serializers""" + +from drf_spectacular.utils import extend_schema_field +from rest_framework import serializers + +from ietf.person.models import Email +from .models import Group, Role + + +class GroupSerializer(serializers.ModelSerializer): + class Meta: + model = Group + fields = ["acronym", "name", "type", "list_email"] + + +class AreaDirectorSerializer(serializers.Serializer): + """Serialize an area director + + Works with Email or Role + """ + + name = serializers.SerializerMethodField() + email = serializers.SerializerMethodField() + + @extend_schema_field(serializers.CharField) + def get_name(self, instance: Email | Role): + person = getattr(instance, 'person', None) + return person.plain_name() if person else None + + @extend_schema_field(serializers.EmailField) + def get_email(self, instance: Email | Role): + if isinstance(instance, Role): + return instance.email.email_address() + return instance.email_address() + + +class AreaSerializer(serializers.ModelSerializer): + ads = serializers.SerializerMethodField() + + class Meta: + model = Group + fields = ["acronym", "name", "ads"] + + @extend_schema_field(AreaDirectorSerializer(many=True)) + def get_ads(self, area: Group): + return AreaDirectorSerializer( + area.ads if area.is_active else Role.objects.none(), + many=True, + ).data diff --git a/ietf/group/tasks.py b/ietf/group/tasks.py index 693aafb385..ada83e80e2 100644 --- a/ietf/group/tasks.py +++ b/ietf/group/tasks.py @@ -9,12 +9,15 @@ from django.conf import settings from django.template.loader import render_to_string +from django.utils import timezone from ietf.doc.storage_utils import store_file +from ietf.liaisons.models import LiaisonStatement from ietf.utils import log +from ietf.utils.test_runner import disable_coverage -from .models import Group -from .utils import fill_in_charter_info, fill_in_wg_drafts, fill_in_wg_roles +from .models import Group, GroupHistory +from .utils import fill_in_charter_info, fill_in_wg_drafts, fill_in_wg_roles, save_group_in_history from .views import extract_last_name, roles @@ -113,3 +116,117 @@ def generate_wg_summary_files_task(): store_file("indexes", "1wg-summary.txt", f, allow_overwrite=True) with summary_by_acronym_file.open("rb") as f: store_file("indexes", "1wg-summary-by-acronym.txt", f, allow_overwrite=True) + +@shared_task +@disable_coverage() +def run_once_adjust_liaison_groups(): # pragma: no cover + log.log("Starting run_once_adjust_liaison_groups") + if all( + [ + Group.objects.filter( + acronym__in=[ + "3gpp-tsg-ct", + "3gpp-tsg-ran-wg1", + "3gpp-tsg-ran-wg4", + "3gpp-tsg-sa", + "3gpp-tsg-sa-wg5", + "3gpp-tsgct", # duplicates 3gpp-tsg-ct above already + "3gpp-tsgct-ct1", # will normalize all acronyms to hyphenated form + "3gpp-tsgct-ct3", # and consistently match the name + "3gpp-tsgct-ct4", # (particularly use of WG) + "3gpp-tsgran", + "3gpp-tsgran-ran2", + "3gpp-tsgsa", # duplicates 3gpp-tsg-sa above + "3gpp-tsgsa-sa2", # will normalize + "3gpp-tsgsa-sa3", + "3gpp-tsgsa-sa4", + "3gpp-tsgt-wg2", + ] + ).count() + == 16, + not Group.objects.filter( + acronym__in=[ + "3gpp-tsg-ran-wg3", + "3gpp-tsg-ct-wg1", + "3gpp-tsg-ct-wg3", + "3gpp-tsg-ct-wg4", + "3gpp-tsg-ran", + "3gpp-tsg-ran-wg2", + "3gpp-tsg-sa-wg2", + "3gpp-tsg-sa-wg3", + "3gpp-tsg-sa-wg4", + "3gpp-tsg-t-wg2", + ] + ).exists(), + Group.objects.filter(acronym="o3gpptsgran3").exists(), + not LiaisonStatement.objects.filter( + to_groups__acronym__in=["3gpp-tsgct", "3gpp-tsgsa"] + ).exists(), + not LiaisonStatement.objects.filter( + from_groups__acronym="3gpp-tsgct" + ).exists(), + LiaisonStatement.objects.filter(from_groups__acronym="3gpp-tsgsa").count() + == 1, + LiaisonStatement.objects.get(from_groups__acronym="3gpp-tsgsa").pk == 1448, + ] + ): + for old_acronym, new_acronym, new_name in ( + ("o3gpptsgran3", "3gpp-tsg-ran-wg3", "3GPP TSG RAN WG3"), + ("3gpp-tsgct-ct1", "3gpp-tsg-ct-wg1", "3GPP TSG CT WG1"), + ("3gpp-tsgct-ct3", "3gpp-tsg-ct-wg3", "3GPP TSG CT WG3"), + ("3gpp-tsgct-ct4", "3gpp-tsg-ct-wg4", "3GPP TSG CT WG4"), + ("3gpp-tsgran", "3gpp-tsg-ran", "3GPP TSG RAN"), + ("3gpp-tsgran-ran2", "3gpp-tsg-ran-wg2", "3GPP TSG RAN WG2"), + ("3gpp-tsgsa-sa2", "3gpp-tsg-sa-wg2", "3GPP TSG SA WG2"), + ("3gpp-tsgsa-sa3", "3gpp-tsg-sa-wg3", "3GPP TSG SA WG3"), + ("3gpp-tsgsa-sa4", "3gpp-tsg-sa-wg4", "3GPP TSG SA WG4"), + ("3gpp-tsgt-wg2", "3gpp-tsg-t-wg2", "3GPP TSG T WG2"), + ): + group = Group.objects.get(acronym=old_acronym) + save_group_in_history(group) + group.time = timezone.now() + group.acronym = new_acronym + group.name = new_name + if old_acronym.startswith("3gpp-tsgct-"): + group.parent = Group.objects.get(acronym="3gpp-tsg-ct") + elif old_acronym.startswith("3gpp-tsgsa-"): + group.parent = Group.objects.get(acronym="3gpp-tsg-sa") + group.save() + group.groupevent_set.create( + time=group.time, + by_id=1, # (System) + type="info_changed", + desc=f"acronym changed from {old_acronym} to {new_acronym}, name set to {new_name}", + ) + + for acronym, new_name in (("3gpp-tsg-ct", "3GPP TSG CT"),): + group = Group.objects.get(acronym=acronym) + save_group_in_history(group) + group.time = timezone.now() + group.name = new_name + group.save() + group.groupevent_set.create( + time=group.time, + by_id=1, # (System) + type="info_changed", + desc=f"name set to {new_name}", + ) + + ls = LiaisonStatement.objects.get(pk=1448) + ls.from_groups.remove(Group.objects.get(acronym="3gpp-tsgsa")) + ls.from_groups.add(Group.objects.get(acronym="3gpp-tsg-sa")) + + # Rewriting history to effectively merge the histories of the duplicate groups + GroupHistory.objects.filter(parent__acronym="3gpp-tsgsa").update( + parent=Group.objects.get(acronym="3gpp-tsg-sa") + ) + GroupHistory.objects.filter(parent__acronym="3gpp-tsgct").update( + parent=Group.objects.get(acronym="3gpp-tsg-ct") + ) + + deleted = Group.objects.filter( + acronym__in=["3gpp-tsgsa", "3gpp-tsgct"] + ).delete() + log.log(f"Deleted Groups: {deleted}") + else: + log.log("* Refusing to continue as preconditions have changed") diff --git a/ietf/group/templatetags/group_filters.py b/ietf/group/templatetags/group_filters.py index c9481b767b..bf2ad71949 100644 --- a/ietf/group/templatetags/group_filters.py +++ b/ietf/group/templatetags/group_filters.py @@ -37,3 +37,10 @@ def role_person_link(role, **kwargs): plain_name = role.person.plain_name() email = role.email.address return {'name': name, 'plain_name': plain_name, 'email': email, 'title': title, 'class': cls} + +@register.filter +def name_with_conditional_acronym(group): + if group.type_id in ("sdo", "isoc", "individ", "nomcom", "ietf", "irtf", ): + return group.name + else: + return f"{group.name} ({group.acronym})" diff --git a/ietf/group/tests.py b/ietf/group/tests.py index 31f8cc45b5..229744388c 100644 --- a/ietf/group/tests.py +++ b/ietf/group/tests.py @@ -3,7 +3,7 @@ import datetime import json -import mock +from unittest import mock from django.urls import reverse as urlreverse from django.db.models import Q diff --git a/ietf/group/tests_info.py b/ietf/group/tests_info.py index aaf937ee43..3f24e2e3d6 100644 --- a/ietf/group/tests_info.py +++ b/ietf/group/tests_info.py @@ -6,7 +6,7 @@ import datetime import io import bleach -import mock +from unittest import mock from unittest.mock import call, patch from pathlib import Path @@ -27,7 +27,7 @@ from ietf.community.models import CommunityList from ietf.community.utils import reset_name_contains_index_for_rule -from ietf.doc.factories import WgDraftFactory, IndividualDraftFactory, CharterFactory, BallotDocEventFactory +from ietf.doc.factories import WgDraftFactory, RgDraftFactory, IndividualDraftFactory, CharterFactory, BallotDocEventFactory from ietf.doc.models import Document, DocEvent, State from ietf.doc.storage_utils import retrieve_str from ietf.doc.utils_charter import charter_name_for_group @@ -413,6 +413,7 @@ def test_group_documents(self): self.assertContains(r, draft3.name) for ah in draft3.action_holders.all(): self.assertContains(r, escape(ah.name)) + self.assertContains(r, "Active with the IESG Internet-Draft") # draft3 is pub-req hence should have such a divider self.assertContains(r, 'for 173 days', count=1) # the old_dah should be tagged self.assertContains(r, draft4.name) self.assertNotContains(r, draft5.name) @@ -425,6 +426,25 @@ def test_group_documents(self): q = PyQuery(r.content) self.assertTrue(any([draft2.name in x.attrib['href'] for x in q('table td a.track-untrack-doc')])) + # Let's also check the IRTF stream + rg = GroupFactory(type_id='rg') + setup_default_community_list_for_group(rg) + rgDraft = RgDraftFactory(group=rg) + rgDraft4 = RgDraftFactory(group=rg) + rgDraft4.set_state(State.objects.get(slug='irsg-w')) + rgDraft7 = RgDraftFactory(group=rg) + rgDraft7.set_state(State.objects.get(type='draft-stream-%s' % rgDraft7.stream_id, slug='dead')) + for url in group_urlreverse_list(rg, 'ietf.group.views.group_documents'): + with self.settings(DOC_ACTION_HOLDER_MAX_AGE_DAYS=20): + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertContains(r, rgDraft.name) + self.assertContains(r, rg.name) + self.assertContains(r, rg.acronym) + self.assertNotContains(r, draft3.name) # As draft3 is a WG draft, it should not be listed here + self.assertContains(r, rgDraft4.name) + self.assertNotContains(r, rgDraft7.name) + # test the txt version too while we're at it for url in group_urlreverse_list(group, 'ietf.group.views.group_documents_txt'): r = self.client.get(url) @@ -523,6 +543,25 @@ def verify_can_edit_group(url, group, username): for username in list(set(interesting_users)-set(can_edit[group.type_id])): verify_cannot_edit_group(url, group, username) + def test_group_about_team_parent(self): + """Team about page should show parent when parent is not an area""" + GroupFactory(type_id='team', parent=GroupFactory(type_id='area', acronym='gen')) + GroupFactory(type_id='team', parent=GroupFactory(type_id='ietf', acronym='iab')) + GroupFactory(type_id='team', parent=None) + + for team in Group.objects.filter(type='team').select_related('parent'): + url = urlreverse('ietf.group.views.group_about', kwargs=dict(acronym=team.acronym)) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + if team.parent and team.parent.type_id != 'area': + self.assertContains(r, 'Parent') + self.assertContains(r, team.parent.acronym) + elif team.parent and team.parent.type_id == 'area': + self.assertContains(r, team.parent.name) + self.assertNotContains(r, '>Parent<') + else: + self.assertNotContains(r, '>Parent<') + def test_group_about_personnel(self): """Correct personnel should appear on the group About page""" group = GroupFactory() diff --git a/ietf/group/tests_review.py b/ietf/group/tests_review.py index a03b806f8f..bb9b79a416 100644 --- a/ietf/group/tests_review.py +++ b/ietf/group/tests_review.py @@ -815,3 +815,170 @@ def test_reset_next_reviewer(self): self.assertEqual(NextReviewerInTeam.objects.get(team=group).next_reviewer, reviewers[target_index].person) self.client.logout() target_index += 2 + +class RequestsHistoryTests(TestCase): + def test_requests_history_overview_page(self): + # Make assigned assignment + review_req = ReviewRequestFactory(state_id='assigned') + assignment = ReviewAssignmentFactory(review_request=review_req, + state_id='assigned', + reviewer=EmailFactory(), + assigned_on = review_req.time) + group = review_req.team + + for url in [urlreverse(ietf.group.views.review_requests_history, + kwargs={ 'acronym': group.acronym }), + urlreverse(ietf.group.views.review_requests_history, + kwargs={ 'acronym': group.acronym , + 'group_type': group.type_id}), + urlreverse(ietf.group.views.review_requests_history, + kwargs={ 'acronym': group.acronym }) + + '?since=3m', + urlreverse(ietf.group.views.review_requests_history, + kwargs={ 'acronym': group.acronym , + 'group_type': group.type_id }) + + '?since=3m']: + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertContains(r, review_req.doc.name) + self.assertContains(r, 'Assigned') + self.assertContains(r, escape(assignment.reviewer.person.name)) + + url = urlreverse(ietf.group.views.review_requests_history, + kwargs={ 'acronym': group.acronym }) + + assignment.state = ReviewAssignmentStateName.objects.get(slug="completed") + assignment.result = ReviewResultName.objects.get(slug="ready") + assignment.save() + + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertContains(r, review_req.doc.name) + self.assertContains(r, 'Assigned') + self.assertContains(r, 'Completed') + + def test_requests_history_filter_page(self): + # First assignment as assigned + review_req = ReviewRequestFactory(state_id = 'assigned', + doc = DocumentFactory()) + assignment = ReviewAssignmentFactory(review_request = review_req, + state_id = 'assigned', + reviewer = EmailFactory(), + assigned_on = review_req.time) + group = review_req.team + + # Second assignment in same group as accepted + review_req2 = ReviewRequestFactory(state_id = 'assigned', + team = review_req.team, + doc = DocumentFactory()) + assignment2 = ReviewAssignmentFactory(review_request = review_req2, + state_id='accepted', + reviewer = EmailFactory(), + assigned_on = review_req2.time) + + # Modify the assignment to be completed, and mark it ready + assignment2.state = ReviewAssignmentStateName.objects.get(slug="completed") + assignment2.result = ReviewResultName.objects.get(slug="ready") + assignment2.save() + + # Check that we have all information when we do not filter + url = urlreverse(ietf.group.views.review_requests_history, + kwargs={ 'acronym': group.acronym }) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertContains(r, review_req.doc.name) + self.assertContains(r, review_req2.doc.name) + self.assertContains(r, 'data-text="Assigned"') + self.assertContains(r, 'data-text="Accepted"') + self.assertContains(r, 'data-text="Completed"') + self.assertContains(r, 'data-text="Ready"') + self.assertContains(r, escape(assignment.reviewer.person.name)) + self.assertContains(r, escape(assignment2.reviewer.person.name)) + + # Check first reviewer history + for url in [urlreverse(ietf.group.views.review_requests_history, + kwargs={ 'acronym': group.acronym }) + + '?reviewer_email=' + str(assignment.reviewer), + urlreverse(ietf.group.views.review_requests_history, + kwargs={ 'acronym': group.acronym , + 'group_type': group.type_id}) + + '?reviewer_email=' + str(assignment.reviewer)]: + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertContains(r, review_req.doc.name) + self.assertNotContains(r, review_req2.doc.name) + self.assertContains(r, 'data-text="Assigned"') + self.assertNotContains(r, 'data-text="Accepted"') + self.assertNotContains(r, 'data-text="Completed"') + self.assertNotContains(r, 'data-text="Ready"') + self.assertContains(r, escape(assignment.reviewer.person.name)) + self.assertNotContains(r, escape(assignment2.reviewer.person.name)) + + # Check second reviewer history + for url in [urlreverse(ietf.group.views.review_requests_history, + kwargs={ 'acronym': group.acronym }) + + '?reviewer_email=' + str(assignment2.reviewer), + urlreverse(ietf.group.views.review_requests_history, + kwargs={ 'acronym': group.acronym , + 'group_type': group.type_id}) + + '?reviewer_email=' + str(assignment2.reviewer)]: + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertNotContains(r, review_req.doc.name) + self.assertContains(r, review_req2.doc.name) + self.assertNotContains(r, 'data-text="Assigned"') + self.assertContains(r, 'data-text="Accepted"') + self.assertContains(r, 'data-text="Completed"') + self.assertContains(r, 'data-text="Ready"') + self.assertNotContains(r, escape(assignment.reviewer.person.name)) + self.assertContains(r, escape(assignment2.reviewer.person.name)) + + # Check for reviewer that does not have anything + url = urlreverse(ietf.group.views.review_requests_history, + kwargs={ 'acronym': group.acronym }) + '?reviewer_email=nobody@nowhere.example.org' + + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertNotContains(r, review_req.doc.name) + self.assertNotContains(r, 'data-text="Assigned"') + self.assertNotContains(r, 'data-text="Accepted"') + self.assertNotContains(r, 'data-text="Completed"') + + def test_requests_history_invalid_filter_parameters(self): + # First assignment as assigned + review_req = ReviewRequestFactory(state_id="assigned", doc=DocumentFactory()) + group = review_req.team + url = urlreverse( + "ietf.group.views.review_requests_history", + kwargs={"acronym": group.acronym}, + ) + invalid_reviewer_emails = [ + "%00null@example.com", # urlencoded null character + "null@exa%00mple.com", # urlencoded null character + "\x00null@example.com", # literal null character + "null@ex\x00ample.com", # literal null character + ] + for invalid_email in invalid_reviewer_emails: + r = self.client.get( + url + f"?reviewer_email={invalid_email}" + ) + self.assertEqual( + r.status_code, + 400, + f"should return a 400 response for reviewer_email={repr(invalid_email)}" + ) + + invalid_since_choices = [ + "forever", # not an option + "all\x00", # literal null character + "a%00ll", # urlencoded null character + ] + for invalid_since in invalid_since_choices: + r = self.client.get( + url + f"?since={invalid_since}" + ) + self.assertEqual( + r.status_code, + 400, + f"should return a 400 response for since={repr(invalid_since)}" + ) diff --git a/ietf/group/tests_serializers.py b/ietf/group/tests_serializers.py new file mode 100644 index 0000000000..b584a17ae2 --- /dev/null +++ b/ietf/group/tests_serializers.py @@ -0,0 +1,96 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +from ietf.group.factories import RoleFactory, GroupFactory +from ietf.group.serializers import ( + AreaDirectorSerializer, + AreaSerializer, + GroupSerializer, +) +from ietf.person.factories import EmailFactory +from ietf.utils.test_utils import TestCase + + +class GroupSerializerTests(TestCase): + def test_serializes(self): + wg = GroupFactory() + serialized = GroupSerializer(wg).data + self.assertEqual( + serialized, + { + "acronym": wg.acronym, + "name": wg.name, + "type": "wg", + "list_email": wg.list_email, + }, + ) + + +class AreaDirectorSerializerTests(TestCase): + def test_serializes_role(self): + """Should serialize a Role correctly""" + role = RoleFactory(group__type_id="area", name_id="ad") + serialized = AreaDirectorSerializer(role).data + self.assertEqual( + serialized, + {"email": role.email.email_address(), "name": role.person.plain_name()}, + ) + + def test_serializes_email(self): + """Should serialize an Email correctly""" + email = EmailFactory() + serialized = AreaDirectorSerializer(email).data + self.assertEqual( + serialized, + { + "email": email.email_address(), + "name": email.person.plain_name() if email.person else None, + }, + ) + + +class AreaSerializerTests(TestCase): + def test_serializes_active_area(self): + """Should serialize an active area correctly""" + area = GroupFactory(type_id="area", state_id="active") + serialized = AreaSerializer(area).data + self.assertEqual( + serialized, + { + "acronym": area.acronym, + "name": area.name, + "ads": [], + }, + ) + ad_roles = RoleFactory.create_batch(2, group=area, name_id="ad") + serialized = AreaSerializer(area).data + self.assertEqual(serialized["acronym"], area.acronym) + self.assertEqual(serialized["name"], area.name) + self.assertCountEqual( + serialized["ads"], + [ + {"email": ad.email.email_address(), "name": ad.person.plain_name()} + for ad in ad_roles + ], + ) + + def test_serializes_inactive_area(self): + """Should serialize an inactive area correctly""" + area = GroupFactory(type_id="area", state_id="conclude") + serialized = AreaSerializer(area).data + self.assertEqual( + serialized, + { + "acronym": area.acronym, + "name": area.name, + "ads": [], + }, + ) + RoleFactory.create_batch(2, group=area, name_id="ad") + serialized = AreaSerializer(area).data + self.assertEqual( + serialized, + { + "acronym": area.acronym, + "name": area.name, + "ads": [], + }, + ) diff --git a/ietf/group/urls.py b/ietf/group/urls.py index 1824564c4d..8354aba063 100644 --- a/ietf/group/urls.py +++ b/ietf/group/urls.py @@ -24,6 +24,7 @@ url(r'^about/status/edit/$', views.group_about_status_edit), url(r'^about/status/meeting/(?P\d+)/$', views.group_about_status_meeting), url(r'^history/$',views.history), + url(r'^requestshistory/$',views.review_requests_history), url(r'^history/addcomment/$',views.add_comment), url(r'^email/$', views.email), url(r'^deps\.json$', views.dependencies), diff --git a/ietf/group/utils.py b/ietf/group/utils.py index dcf9d83e6f..6777ed1933 100644 --- a/ietf/group/utils.py +++ b/ietf/group/utils.py @@ -236,9 +236,11 @@ def construct_group_menu_context(request, group, selected, group_type, others): import ietf.group.views entries.append(("Review requests", urlreverse(ietf.group.views.review_requests, kwargs=kwargs))) entries.append(("Reviewers", urlreverse(ietf.group.views.reviewer_overview, kwargs=kwargs))) - + entries.append(("Reviews History", urlreverse(ietf.group.views.review_requests_history, kwargs=kwargs))) if group.features.has_meetings: entries.append(("Meetings", urlreverse("ietf.group.views.meetings", kwargs=kwargs))) + if group.acronym in ["iesg"]: + entries.append(("Working Groups", urlreverse("ietf.iesg.views.working_groups"))) if group.acronym in ["iab", "iesg"]: entries.append(("Statements", urlreverse("ietf.group.views.statements", kwargs=kwargs))) entries.append(("Appeals", urlreverse("ietf.group.views.appeals", kwargs=kwargs))) @@ -249,7 +251,6 @@ def construct_group_menu_context(request, group, selected, group_type, others): if is_valid_url(group.list_archive): entries.append((mark_safe("List archive »"), group.list_archive)) - # actions actions = [] diff --git a/ietf/group/views.py b/ietf/group/views.py index f30569d230..8561a5059f 100644 --- a/ietf/group/views.py +++ b/ietf/group/views.py @@ -51,13 +51,20 @@ from django.contrib.auth.decorators import login_required from django.db.models import Count, F, OuterRef, Prefetch, Q, Subquery, TextField, Value from django.db.models.functions import Coalesce -from django.http import HttpResponse, HttpResponseRedirect, Http404, JsonResponse +from django.http import ( + HttpResponse, + HttpResponseRedirect, + Http404, + JsonResponse, + HttpResponseBadRequest, +) from django.shortcuts import render, redirect, get_object_or_404 from django.template.loader import render_to_string from django.urls import reverse as urlreverse from django.utils import timezone from django.utils.html import escape from django.views.decorators.cache import cache_page, cache_control +from django.urls import reverse import debug # pyflakes:ignore @@ -95,11 +102,9 @@ from ietf.review.policies import get_reviewer_queue_policy from ietf.review.utils import (can_manage_review_requests_for_team, can_access_review_stats_for_team, - extract_revision_ordered_review_requests_for_documents_and_replaced, assign_review_request_to_reviewer, close_review_request, - suggested_review_requests_for_team, unavailable_periods_to_list, current_unavailable_periods_for_reviewers, @@ -240,10 +245,19 @@ def active_review_dirs(request): return render(request, 'group/active_review_dirs.html', {'dirs' : dirs }) def active_teams(request): - teams = Group.objects.filter(type="team", state="active").order_by("name") + parent_type_order = {"area": 1, "adm": 3, None: 4} + + def team_sort_key(group): + type_id = group.parent.type_id if group.parent else None + return (parent_type_order.get(type_id, 2), group.parent.name if group.parent else "", group.name) + + teams = sorted( + Group.objects.filter(type="team", state="active").select_related("parent"), + key=team_sort_key, + ) for group in teams: group.chairs = sorted(roles(group, "chair"), key=extract_last_name) - return render(request, 'group/active_teams.html', {'teams' : teams }) + return render(request, 'group/active_teams.html', {'teams': teams}) def active_iab(request): iabgroups = Group.objects.filter(type__in=("program","iabasg","iabworkshop"), state="active").order_by("-type_id","name") @@ -438,7 +452,6 @@ def prepare_group_documents(request, group, clist): return docs, meta, docs_related, meta_related - def get_leadership(group_type): people = Person.objects.filter( role__name__slug="chair", @@ -685,6 +698,61 @@ def history(request, acronym, group_type=None): "can_add_comment": can_add_comment, })) + +class RequestsHistoryParamsForm(forms.Form): + SINCE_CHOICES = ( + (None, "1 month"), + ("3m", "3 months"), + ("6m", "6 months"), + ("1y", "1 year"), + ("2y", "2 years"), + ("all", "All"), + ) + + reviewer_email = forms.EmailField(required=False) + since = forms.ChoiceField(choices=SINCE_CHOICES, required=False) + +def review_requests_history(request, acronym, group_type=None): + group = get_group_or_404(acronym, group_type) + if not group.features.has_reviews: + raise Http404 + + params = RequestsHistoryParamsForm(request.GET) + if not params.is_valid(): + return HttpResponseBadRequest("Invalid parameters") + + reviewer_email = params.cleaned_data["reviewer_email"] or None + if reviewer_email: + history = ReviewAssignment.history.model.objects.filter( + review_request__team__acronym=acronym, + reviewer=reviewer_email) + else: + history = ReviewAssignment.history.model.objects.filter( + review_request__team__acronym=acronym) + reviewer_email = '' + + since = params.cleaned_data["since"] or None + if since != "all": + date_limit = { + None: datetime.timedelta(days=31), + "3m": datetime.timedelta(days=31 * 3), + "6m": datetime.timedelta(days=180), + "1y": datetime.timedelta(days=365), + "2y": datetime.timedelta(days=2 * 365), + }[since] + + history = history.filter(review_request__time__gte=datetime_today(DEADLINE_TZINFO) - date_limit) + + return render(request, 'group/review_requests_history.html', + construct_group_menu_context(request, group, "reviews history", group_type, { + "group": group, + "acronym": acronym, + "history": history, + "since_choices": params.SINCE_CHOICES, + "since": since, + "reviewer_email": reviewer_email + })) + def materials(request, acronym, group_type=None): group = get_group_or_404(acronym, group_type) if not group.features.has_nonsession_materials: @@ -881,7 +949,7 @@ def meetings(request, acronym, group_type=None): cutoff_date = revsub_dates_by_meeting[s.meeting.pk] else: cutoff_date = s.meeting.date + datetime.timedelta(days=s.meeting.submission_correction_day_offset) - s.cached_is_cutoff = date_today(datetime.timezone.utc) > cutoff_date + s.cached_is_cutoff = date_today(datetime.UTC) > cutoff_date future, in_progress, recent, past = group_sessions(sessions) @@ -898,6 +966,18 @@ def meetings(request, acronym, group_type=None): far_past.append(s) past = recent_past + # Add calendar actions + cal_actions = [] + + cal_actions.append(dict( + label='Download as .ics', + url=reverse('ietf.meeting.views.upcoming_ical')+"?show="+group.acronym) + ) + cal_actions.append(dict( + label='Subscribe with webcal', + url='webcal://'+request.get_host()+reverse('ietf.meeting.views.upcoming_ical')+"?show="+group.acronym) + ) + return render( request, "group/meetings.html", @@ -915,6 +995,7 @@ def meetings(request, acronym, group_type=None): "far_past": far_past, "can_edit": can_edit, "can_always_edit": can_always_edit, + "cal_actions": cal_actions, }, ), ) @@ -2187,7 +2268,7 @@ def statements(request, acronym, group_type=None): ).values_list("state__slug", flat=True)[:1] ) ) - .order_by("-published") + .order_by("status", "-published") ) return render( request, diff --git a/ietf/idindex/index.py b/ietf/idindex/index.py index 4f021c0dc7..19eb29d4da 100644 --- a/ietf/idindex/index.py +++ b/ietf/idindex/index.py @@ -276,7 +276,7 @@ def active_drafts_index_by_group(extra_values=()): groups = [g for g in groups_dict.values() if hasattr(g, "active_drafts")] groups.sort(key=lambda g: g.acronym) - fallback_time = datetime.datetime(1950, 1, 1, tzinfo=datetime.timezone.utc) + fallback_time = datetime.datetime(1950, 1, 1, tzinfo=datetime.UTC) for g in groups: g.active_drafts.sort(key=lambda d: d.get("initial_rev_time", fallback_time)) @@ -302,6 +302,6 @@ def id_index_txt(with_abstracts=False): return render_to_string("idindex/id_index.txt", { 'groups': groups, - 'time': timezone.now().astimezone(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S %Z"), + 'time': timezone.now().astimezone(datetime.UTC).strftime("%Y-%m-%d %H:%M:%S %Z"), 'with_abstracts': with_abstracts, }) diff --git a/ietf/idindex/tests.py b/ietf/idindex/tests.py index 5cc7a7b3bb..ba6100550d 100644 --- a/ietf/idindex/tests.py +++ b/ietf/idindex/tests.py @@ -3,7 +3,7 @@ import datetime -import mock +from unittest import mock from pathlib import Path from tempfile import TemporaryDirectory diff --git a/ietf/iesg/agenda.py b/ietf/iesg/agenda.py index 587713089f..ace4c9ec40 100644 --- a/ietf/iesg/agenda.py +++ b/ietf/iesg/agenda.py @@ -133,7 +133,7 @@ def agenda_sections(): ('4.2', {'title':"WG rechartering"}), ('4.2.1', {'title':"Under evaluation for IETF review", 'docs':[]}), ('4.2.2', {'title':"Proposed for approval", 'docs':[]}), - ('5', {'title':"IAB news we can use"}), + ('5', {'title':"IESG Liaison News"}), ('6', {'title':"Management issues"}), ('7', {'title':"Any Other Business (WG News, New Proposals, etc.)"}), ]) diff --git a/ietf/iesg/tests.py b/ietf/iesg/tests.py index 86910bc0ce..e5fbe5da7b 100644 --- a/ietf/iesg/tests.py +++ b/ietf/iesg/tests.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- +from collections import Counter import datetime import io import tarfile @@ -24,7 +25,9 @@ from ietf.group.models import Group, GroupMilestone, Role from ietf.iesg.agenda import get_agenda_date, agenda_data, fill_in_agenda_administrivia, agenda_sections from ietf.iesg.models import TelechatDate, TelechatAgendaContent +from ietf.iesg.utils import get_wg_dashboard_info from ietf.name.models import StreamName, TelechatAgendaSectionName +from ietf.person.factories import PersonFactory from ietf.person.models import Person from ietf.utils.test_utils import TestCase, login_testing_unauthorized, unicontent from ietf.iesg.factories import IESGMgmtItemFactory, TelechatAgendaContentFactory @@ -182,6 +185,1587 @@ def test_ietf_activity(self): r = self.client.get(url) self.assertEqual(r.status_code, 200) + def test_working_groups(self): + # Clean away the wasted built-for-every-test noise + Group.objects.filter(type__in=["wg", "area"]).delete() + + ( + area_summary, + area_totals, + ad_summary, + noad_summary, + ad_totals, + noad_totals, + totals, + wg_summary, + ) = get_wg_dashboard_info() + self.assertEqual(area_summary, []) + self.assertEqual( + area_totals, {"group_count": 0, "doc_count": 0, "page_count": 0} + ) + self.assertEqual(ad_summary, []) + self.assertEqual(noad_summary, []) + self.assertEqual( + ad_totals, + { + "ad_group_count": 0, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + }, + ) + self.assertEqual( + noad_totals, + { + "ad_group_count": 0, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + }, + ) + self.assertEqual( + totals, + { + "group_count": 0, + "doc_count": 0, + "page_count": 0, + "groups_with_docs_count": 0, + }, + ) + self.assertEqual(wg_summary, []) + + # Construct Areas with WGs similar in shape to a real moment of the IETF + + # Note that this test construciton uses the first letter of the wg acronyms + # for convenience to switch on whether groups have documents with assigned ADs. + # (Search for ` if wg_acronym[0] > "g"`) + # There's no other significance to the names of the area directors or the + # acronyms of the areas and groups other than being distinct. Taking the + # values from sets of similar things hopefully helps with debugging the tests. + + areas = {} + for area_acronym in ["red", "orange", "yellow", "green", "blue", "violet"]: + areas[area_acronym] = GroupFactory(type_id="area", acronym=area_acronym) + for ad, area, wgs in [ + ("Alpha", "red", ["bassoon"]), + ("Bravo", "orange", ["celesta"]), + ("Charlie", "orange", ["clarinet", "cymbals"]), + ("Delta", "yellow", ["flute"]), + ("Echo", "yellow", ["glockenspiel"]), + ("Foxtrot", "green", ["gong", "guitar"]), + ("Golf", "green", ["harp"]), + ("Hotel", "blue", ["harpsichord"]), + ("Indigo", "blue", ["oboe", "organ"]), + ("Juliet", "violet", ["piano"]), + ("Kilo", "violet", ["piccolo"]), + ("Lima", "violet", ["saxophone", "tambourine"]), + ]: + p = Person.objects.filter(name=ad).first() or PersonFactory(name=ad) + RoleFactory(group=areas[area], person=p, name_id="ad") + for wg in wgs: + g = GroupFactory(acronym=wg, type_id="wg", parent=areas[area]) + RoleFactory(group=g, person=p, name_id="ad") + + # Some ADs have out of area groups + g = GroupFactory(acronym="timpani", parent=areas["orange"]) + RoleFactory(group=g, person=Person.objects.get(name="Juliet"), name_id="ad") + + ( + area_summary, + area_totals, + ad_summary, + noad_summary, + ad_totals, + noad_totals, + totals, + wg_summary, + ) = get_wg_dashboard_info() + + # checks for the expected result with area sorted by name + self.assertEqual( + area_summary, + [ + { + "area": "blue", + "groups_in_area": 3, + "groups_with_docs": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "area": "green", + "groups_in_area": 3, + "groups_with_docs": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "area": "orange", + "groups_in_area": 4, + "groups_with_docs": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "area": "red", + "groups_in_area": 1, + "groups_with_docs": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "area": "violet", + "groups_in_area": 4, + "groups_with_docs": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "area": "yellow", + "groups_in_area": 2, + "groups_with_docs": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + ], + ) + self.assertEqual( + area_totals, {"group_count": 0, "doc_count": 0, "page_count": 0} + ) + self.assertEqual( + ad_summary, + [ + { + "ad": "Alpha", + "area": "red", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Bravo", + "area": "orange", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Charlie", + "area": "orange", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Delta", + "area": "yellow", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Echo", + "area": "yellow", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Foxtrot", + "area": "green", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Golf", + "area": "green", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Hotel", + "area": "blue", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Indigo", + "area": "blue", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Juliet", + "area": "orange", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Juliet", + "area": "violet", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Kilo", + "area": "violet", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Lima", + "area": "violet", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + ], + ) + self.assertEqual( + noad_summary, + [ + { + "ad": "Alpha", + "area": "red", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Bravo", + "area": "orange", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Charlie", + "area": "orange", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Delta", + "area": "yellow", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Echo", + "area": "yellow", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Foxtrot", + "area": "green", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Golf", + "area": "green", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Hotel", + "area": "blue", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Indigo", + "area": "blue", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Juliet", + "area": "orange", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Juliet", + "area": "violet", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Kilo", + "area": "violet", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Lima", + "area": "violet", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + ], + ) + self.assertEqual( + ad_totals, + { + "ad_group_count": 17, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + }, + ) + self.assertEqual( + noad_totals, + { + "ad_group_count": 17, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + }, + ) + self.assertEqual( + totals, + { + "group_count": 17, + "doc_count": 0, + "page_count": 0, + "groups_with_docs_count": 0, + }, + ) + self.assertEqual( + wg_summary, + [ + { + "wg": "bassoon", + "area": "red", + "ad": "Alpha", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "celesta", + "area": "orange", + "ad": "Bravo", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "clarinet", + "area": "orange", + "ad": "Charlie", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "cymbals", + "area": "orange", + "ad": "Charlie", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "flute", + "area": "yellow", + "ad": "Delta", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "glockenspiel", + "area": "yellow", + "ad": "Echo", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "gong", + "area": "green", + "ad": "Foxtrot", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "guitar", + "area": "green", + "ad": "Foxtrot", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "harp", + "area": "green", + "ad": "Golf", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "harpsichord", + "area": "blue", + "ad": "Hotel", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "oboe", + "area": "blue", + "ad": "Indigo", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "organ", + "area": "blue", + "ad": "Indigo", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "piano", + "area": "violet", + "ad": "Juliet", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "piccolo", + "area": "violet", + "ad": "Kilo", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "saxophone", + "area": "violet", + "ad": "Lima", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "tambourine", + "area": "violet", + "ad": "Lima", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "timpani", + "area": "orange", + "ad": "Juliet", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + ], + ) + + # As seen above, all doc and page counts are currently 0 + + # We'll give a group a document but not assign it to its AD + WgDraftFactory( + group=Group.objects.get(acronym="saxophone"), pages=len("saxophone") + ) + ( + area_summary, + area_totals, + ad_summary, + noad_summary, + ad_totals, + noad_totals, + totals, + wg_summary, + ) = get_wg_dashboard_info() + count_violet_dicts = 0 + for d in area_summary: + if d["area"] == "violet": + count_violet_dicts += 1 + self.assertEqual(d["groups_with_docs"], 1) + self.assertEqual(d["doc_count"], 1) + self.assertEqual(d["page_count"], 9) + self.assertEqual(d["group_percent"], 100.0) + self.assertEqual(d["doc_percent"], 100.0) + self.assertEqual(d["page_percent"], 100.0) + else: + self.assertEqual(d["groups_with_docs"], 0) + self.assertEqual(d["doc_count"], 0) + self.assertEqual(d["page_count"], 0) + self.assertEqual(d["group_percent"], 0) + self.assertEqual(d["doc_percent"], 0) + self.assertEqual(d["page_percent"], 0) + self.assertEqual(count_violet_dicts, 1) + + self.assertEqual( + area_totals, {"group_count": 1, "doc_count": 1, "page_count": 9} + ) + + # No AD has this document, even though it's in Lima's group + count_lima_dicts = 0 + for d in ad_summary: + if d["ad"] == "Lima": + count_lima_dicts += 1 + self.assertEqual(d["doc_group_count"], 0) + self.assertEqual(d["doc_count"], 0) + self.assertEqual(d["page_count"], 0) + self.assertEqual(d["group_percent"], 0) + self.assertEqual(d["doc_percent"], 0) + self.assertEqual(d["page_percent"], 0) + self.assertEqual(count_lima_dicts, 1) + + # It's in Lima's group, so normally it will eventually land on Lima + count_lima_dicts = 0 + for d in noad_summary: + if d["ad"] == "Lima": + count_lima_dicts += 1 + self.assertEqual(d["doc_group_count"], 1) + self.assertEqual(d["doc_count"], 1) + self.assertEqual(d["page_count"], 9) + self.assertEqual(d["group_percent"], 100.0) + self.assertEqual(d["doc_percent"], 100.0) + self.assertEqual(d["page_percent"], 100.0) + else: + self.assertEqual(d["doc_group_count"], 0) + self.assertEqual(d["doc_count"], 0) + self.assertEqual(d["page_count"], 0) + self.assertEqual(d["group_percent"], 0) + self.assertEqual(d["doc_percent"], 0) + self.assertEqual(d["page_percent"], 0) + self.assertEqual(count_lima_dicts, 1) + + self.assertEqual( + ad_totals, + { + "ad_group_count": 17, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + }, + ) + self.assertEqual( + noad_totals, + { + "ad_group_count": 17, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 9, + }, + ) + self.assertEqual( + totals, + { + "group_count": 17, + "doc_count": 1, + "page_count": 9, + "groups_with_docs_count": 1, + }, + ) + + count_sax_dicts = 0 + for d in wg_summary: + if d["wg"] == "saxophone": + count_sax_dicts += 1 + self.assertEqual(d["doc_count"], 1) + self.assertEqual(d["page_count"], 9) + else: + self.assertEqual(d["doc_count"], 0) + self.assertEqual(d["page_count"], 0) + self.assertEqual(count_sax_dicts, 1) + + # Assign that doc to Lima + self.assertEqual(Document.objects.count(), 1) + Document.objects.all().update(ad=Person.objects.get(name="Lima")) + ( + area_summary, + area_totals, + ad_summary, + noad_summary, + ad_totals, + noad_totals, + totals, + wg_summary, + ) = get_wg_dashboard_info() + count_violet_dicts = 0 + for d in area_summary: + if d["area"] == "violet": + count_violet_dicts += 1 + self.assertEqual(d["groups_with_docs"], 1) + self.assertEqual(d["doc_count"], 1) + self.assertEqual(d["page_count"], 9) + self.assertEqual(d["group_percent"], 100.0) + self.assertEqual(d["doc_percent"], 100.0) + self.assertEqual(d["page_percent"], 100.0) + else: + self.assertEqual(d["groups_with_docs"], 0) + self.assertEqual(d["doc_count"], 0) + self.assertEqual(d["page_count"], 0) + self.assertEqual(d["group_percent"], 0) + self.assertEqual(d["doc_percent"], 0) + self.assertEqual(d["page_percent"], 0) + self.assertEqual(count_violet_dicts, 1) + + self.assertEqual( + area_totals, {"group_count": 1, "doc_count": 1, "page_count": 9} + ) + + # This time it will show up as a doc assigned to Lima + count_lima_dicts = 0 + for d in ad_summary: + if d["ad"] == "Lima": + count_lima_dicts += 1 + self.assertEqual(d["doc_group_count"], 1) + self.assertEqual(d["doc_count"], 1) + self.assertEqual(d["page_count"], 9) + self.assertEqual(d["group_percent"], 100.0) + self.assertEqual(d["doc_percent"], 100.0) + self.assertEqual(d["page_percent"], 100.0) + else: + self.assertEqual(d["doc_group_count"], 0) + self.assertEqual(d["doc_count"], 0) + self.assertEqual(d["page_count"], 0) + self.assertEqual(d["group_percent"], 0) + self.assertEqual(d["doc_percent"], 0) + self.assertEqual(d["page_percent"], 0) + self.assertEqual(count_lima_dicts, 1) + + # and there will be no noad documents + count_lima_dicts = 0 + for d in noad_summary: + if d["ad"] == "Lima": + count_lima_dicts += 1 + self.assertEqual(d["doc_group_count"], 0) + self.assertEqual(d["doc_count"], 0) + self.assertEqual(d["page_count"], 0) + self.assertEqual(d["group_percent"], 0) + self.assertEqual(d["doc_percent"], 0) + self.assertEqual(d["page_percent"], 0) + self.assertEqual(count_lima_dicts, 1) + + self.assertEqual( + ad_totals, + { + "ad_group_count": 17, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 9, + }, + ) + self.assertEqual( + noad_totals, + { + "ad_group_count": 17, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + }, + ) + self.assertEqual( + totals, + { + "group_count": 17, + "doc_count": 1, + "page_count": 9, + "groups_with_docs_count": 1, + }, + ) + + count_sax_dicts = 0 + for d in wg_summary: + if d["wg"] == "saxophone": + count_sax_dicts += 1 + self.assertEqual(d["doc_count"], 1) + self.assertEqual(d["page_count"], 9) + else: + self.assertEqual(d["doc_count"], 0) + self.assertEqual(d["page_count"], 0) + self.assertEqual(count_sax_dicts, 1) + + # Now give Lima a document in a group that's not in their area: + WgDraftFactory( + group=Group.objects.get(acronym="gong"), + pages=len("gong"), + ad=Person.objects.get(name="Lima"), + ) + ( + area_summary, + area_totals, + ad_summary, + noad_summary, + ad_totals, + noad_totals, + totals, + wg_summary, + ) = get_wg_dashboard_info() + seen_dicts = Counter([d["area"] for d in area_summary]) + for d in areas: + self.assertEqual(seen_dicts[area], 1 if area in ["violet", "green"] else 0) + for d in area_summary: + if d["area"] in ["violet", "green"]: + self.assertEqual(d["doc_count"], 1) + self.assertEqual(d["page_count"], 9 if d["area"] == "violet" else 4) + self.assertEqual(d["group_percent"], 50) + self.assertEqual(d["doc_percent"], 50) + self.assertEqual( + d["page_percent"], + 100 * 9 / 13 if d["area"] == "violet" else 100 * 4 / 13, + ) + else: + self.assertEqual(d["doc_count"], 0) + self.assertEqual(d["page_count"], 0) + self.assertEqual(d["group_percent"], 0) + self.assertEqual(d["doc_percent"], 0) + self.assertEqual(d["page_percent"], 0) + + self.assertEqual( + area_totals, {"group_count": 2, "doc_count": 2, "page_count": 13} + ) + + for d in ad_summary: + if d["ad"] == "Lima": + self.assertEqual(d["doc_group_count"], 1) + self.assertEqual(d["doc_count"], 1) + self.assertEqual(d["page_count"], 9 if d["area"] == "violet" else 4) + self.assertEqual(d["group_percent"], 50) + self.assertEqual(d["doc_percent"], 50) + self.assertEqual( + d["page_percent"], + 100 * 9 / 13 if d["area"] == "violet" else 100 * 4 / 13, + ) + else: + self.assertEqual(d["doc_group_count"], 0) + self.assertEqual( + d["doc_count"], 0 + ) # Note in particular this is 0 for Foxtrot + self.assertEqual(d["page_count"], 0) + self.assertEqual(d["group_percent"], 0) + self.assertEqual(d["doc_percent"], 0) + self.assertEqual(d["page_percent"], 0) + + for d in wg_summary: + if d["wg"] == "gong": + # Lima's doc in gong above counts at the dict for gong even though the ad reported there is Foxtrot. + self.assertEqual( + d, + { + "wg": "gong", + "area": "green", + "ad": "Foxtrot", + "doc_count": 1, + "page_count": 4, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + ) + elif d["ad"] == "Lima": + self.assertEqual( + d["area"], "violet" + ) # The out of area assignment is not reflected in the wg_summary at all. + + # Now pile on a lot of documents + for wg_acronym in [ + "bassoon", + "celesta", + "clarinet", + "cymbals", + "flute", + "glockenspiel", + "gong", + "guitar", + "harp", + "harpsichord", + "oboe", + "organ", + "piano", + "piccolo", + "saxophone", + "tambourine", + "timpani", + ]: + if wg_acronym in ["bassoon", "celesta"]: + continue # Those WGs have no docs + # The rest have a doc that's not assigned to any ad + WgDraftFactory( + group=Group.objects.get(acronym=wg_acronym), pages=len(wg_acronym) + ) + if wg_acronym[0] > "g": + # Some have a doc assigned to the responsible ad + WgDraftFactory( + group=Group.objects.get(acronym=wg_acronym), + pages=len(wg_acronym), + ad=Role.objects.get(name_id="ad", group__acronym=wg_acronym).person, + ) + # The other AD for an area might be covering a doc + WgDraftFactory( + group=Group.objects.get(acronym="saxophone"), + pages=len("saxophone"), + ad=Person.objects.get(name="Juliet"), + ) + # An Ad not associated with the group or the area is responsible for a doc + WgDraftFactory( + group=Group.objects.get(acronym="bassoon"), + pages=len("bassoon"), + ad=Person.objects.get(name="Juliet"), + ) + + ( + area_summary, + area_totals, + ad_summary, + noad_summary, + ad_totals, + noad_totals, + totals, + wg_summary, + ) = get_wg_dashboard_info() + + self.assertEqual( + area_summary, + [ + { + "area": "blue", + "groups_in_area": 3, + "groups_with_docs": 3, + "doc_count": 6, + "page_count": 40, + "group_percent": 18.75, + "doc_percent": 21.428571428571427, + "page_percent": 20.51282051282051, + }, + { + "area": "green", + "groups_in_area": 3, + "groups_with_docs": 3, + "doc_count": 5, + "page_count": 22, + "group_percent": 18.75, + "doc_percent": 17.857142857142858, + "page_percent": 11.282051282051283, + }, + { + "area": "orange", + "groups_in_area": 4, + "groups_with_docs": 3, + "doc_count": 4, + "page_count": 29, + "group_percent": 18.75, + "doc_percent": 14.285714285714285, + "page_percent": 14.871794871794872, + }, + { + "area": "red", + "groups_in_area": 1, + "groups_with_docs": 1, + "doc_count": 1, + "page_count": 7, + "group_percent": 6.25, + "doc_percent": 3.571428571428571, + "page_percent": 3.5897435897435894, + }, + { + "area": "violet", + "groups_in_area": 4, + "groups_with_docs": 4, + "doc_count": 10, + "page_count": 80, + "group_percent": 25.0, + "doc_percent": 35.714285714285715, + "page_percent": 41.02564102564102, + }, + { + "area": "yellow", + "groups_in_area": 2, + "groups_with_docs": 2, + "doc_count": 2, + "page_count": 17, + "group_percent": 12.5, + "doc_percent": 7.142857142857142, + "page_percent": 8.717948717948717, + }, + ], + ) + self.assertEqual( + area_totals, {"group_count": 16, "doc_count": 28, "page_count": 195} + ) + self.assertEqual( + ad_summary, + [ + { + "ad": "Alpha", + "area": "red", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0.0, + "doc_percent": 0.0, + "page_percent": 0.0, + }, + { + "ad": "Bravo", + "area": "orange", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0.0, + "doc_percent": 0.0, + "page_percent": 0.0, + }, + { + "ad": "Charlie", + "area": "orange", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0.0, + "doc_percent": 0.0, + "page_percent": 0.0, + }, + { + "ad": "Delta", + "area": "yellow", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0.0, + "doc_percent": 0.0, + "page_percent": 0.0, + }, + { + "ad": "Echo", + "area": "yellow", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0.0, + "doc_percent": 0.0, + "page_percent": 0.0, + }, + { + "ad": "Foxtrot", + "area": "green", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0.0, + "doc_percent": 0.0, + "page_percent": 0.0, + }, + { + "ad": "Golf", + "area": "green", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 4, + "group_percent": 8.333333333333332, + "doc_percent": 7.6923076923076925, + "page_percent": 4.395604395604396, + }, + { + "ad": "Hotel", + "area": "blue", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 11, + "group_percent": 8.333333333333332, + "doc_percent": 7.6923076923076925, + "page_percent": 12.087912087912088, + }, + { + "ad": "Indigo", + "area": "blue", + "ad_group_count": 2, + "doc_group_count": 2, + "doc_count": 2, + "page_count": 9, + "group_percent": 16.666666666666664, + "doc_percent": 15.384615384615385, + "page_percent": 9.89010989010989, + }, + { + "ad": "Juliet", + "area": "orange", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 7, + "group_percent": 8.333333333333332, + "doc_percent": 7.6923076923076925, + "page_percent": 7.6923076923076925, + }, + { + "ad": "Juliet", + "area": "red", + "ad_group_count": 0, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 7, + "group_percent": 8.333333333333332, + "doc_percent": 7.6923076923076925, + "page_percent": 7.6923076923076925, + }, + { + "ad": "Juliet", + "area": "violet", + "ad_group_count": 1, + "doc_group_count": 2, + "doc_count": 2, + "page_count": 14, + "group_percent": 16.666666666666664, + "doc_percent": 15.384615384615385, + "page_percent": 15.384615384615385, + }, + { + "ad": "Kilo", + "area": "violet", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 7, + "group_percent": 8.333333333333332, + "doc_percent": 7.6923076923076925, + "page_percent": 7.6923076923076925, + }, + { + "ad": "Lima", + "area": "green", + "ad_group_count": 0, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 4, + "group_percent": 8.333333333333332, + "doc_percent": 7.6923076923076925, + "page_percent": 4.395604395604396, + }, + { + "ad": "Lima", + "area": "violet", + "ad_group_count": 2, + "doc_group_count": 2, + "doc_count": 3, + "page_count": 28, + "group_percent": 16.666666666666664, + "doc_percent": 23.076923076923077, + "page_percent": 30.76923076923077, + }, + ], + ) + self.assertEqual( + noad_summary, + [ + { + "ad": "Alpha", + "area": "red", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0.0, + "doc_percent": 0.0, + "page_percent": 0.0, + }, + { + "ad": "Bravo", + "area": "orange", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0.0, + "doc_percent": 0.0, + "page_percent": 0.0, + }, + { + "ad": "Charlie", + "area": "orange", + "ad_group_count": 2, + "doc_group_count": 2, + "doc_count": 2, + "page_count": 15, + "group_percent": 13.333333333333334, + "doc_percent": 13.333333333333334, + "page_percent": 14.423076923076922, + }, + { + "ad": "Delta", + "area": "yellow", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 5, + "group_percent": 6.666666666666667, + "doc_percent": 6.666666666666667, + "page_percent": 4.807692307692308, + }, + { + "ad": "Echo", + "area": "yellow", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 12, + "group_percent": 6.666666666666667, + "doc_percent": 6.666666666666667, + "page_percent": 11.538461538461538, + }, + { + "ad": "Foxtrot", + "area": "green", + "ad_group_count": 2, + "doc_group_count": 2, + "doc_count": 2, + "page_count": 10, + "group_percent": 13.333333333333334, + "doc_percent": 13.333333333333334, + "page_percent": 9.615384615384617, + }, + { + "ad": "Golf", + "area": "green", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 4, + "group_percent": 6.666666666666667, + "doc_percent": 6.666666666666667, + "page_percent": 3.8461538461538463, + }, + { + "ad": "Hotel", + "area": "blue", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 11, + "group_percent": 6.666666666666667, + "doc_percent": 6.666666666666667, + "page_percent": 10.576923076923077, + }, + { + "ad": "Indigo", + "area": "blue", + "ad_group_count": 2, + "doc_group_count": 2, + "doc_count": 2, + "page_count": 9, + "group_percent": 13.333333333333334, + "doc_percent": 13.333333333333334, + "page_percent": 8.653846153846153, + }, + { + "ad": "Juliet", + "area": "orange", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 7, + "group_percent": 6.666666666666667, + "doc_percent": 6.666666666666667, + "page_percent": 6.730769230769231, + }, + { + "ad": "Juliet", + "area": "violet", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 5, + "group_percent": 6.666666666666667, + "doc_percent": 6.666666666666667, + "page_percent": 4.807692307692308, + }, + { + "ad": "Kilo", + "area": "violet", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 7, + "group_percent": 6.666666666666667, + "doc_percent": 6.666666666666667, + "page_percent": 6.730769230769231, + }, + { + "ad": "Lima", + "area": "violet", + "ad_group_count": 2, + "doc_group_count": 2, + "doc_count": 2, + "page_count": 19, + "group_percent": 13.333333333333334, + "doc_percent": 13.333333333333334, + "page_percent": 18.269230769230766, + }, + ], + ) + self.assertEqual( + ad_totals, + { + "ad_group_count": 17, + "doc_group_count": 12, + "doc_count": 13, + "page_count": 91, + }, + ) + self.assertEqual( + noad_totals, + { + "ad_group_count": 17, + "doc_group_count": 15, + "doc_count": 15, + "page_count": 104, + }, + ) + self.assertEqual( + totals, + { + "group_count": 17, + "doc_count": 28, + "page_count": 195, + "groups_with_docs_count": 16, + }, + ) + self.assertEqual( + wg_summary, + [ + { + "wg": "bassoon", + "area": "red", + "ad": "Alpha", + "doc_count": 1, + "page_count": 7, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "celesta", + "area": "orange", + "ad": "Bravo", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "clarinet", + "area": "orange", + "ad": "Charlie", + "doc_count": 1, + "page_count": 8, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "cymbals", + "area": "orange", + "ad": "Charlie", + "doc_count": 1, + "page_count": 7, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "flute", + "area": "yellow", + "ad": "Delta", + "doc_count": 1, + "page_count": 5, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "glockenspiel", + "area": "yellow", + "ad": "Echo", + "doc_count": 1, + "page_count": 12, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "gong", + "area": "green", + "ad": "Foxtrot", + "doc_count": 2, + "page_count": 8, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "guitar", + "area": "green", + "ad": "Foxtrot", + "doc_count": 1, + "page_count": 6, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "harp", + "area": "green", + "ad": "Golf", + "doc_count": 2, + "page_count": 8, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "harpsichord", + "area": "blue", + "ad": "Hotel", + "doc_count": 2, + "page_count": 22, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "oboe", + "area": "blue", + "ad": "Indigo", + "doc_count": 2, + "page_count": 8, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "organ", + "area": "blue", + "ad": "Indigo", + "doc_count": 2, + "page_count": 10, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "piano", + "area": "violet", + "ad": "Juliet", + "doc_count": 2, + "page_count": 10, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "piccolo", + "area": "violet", + "ad": "Kilo", + "doc_count": 2, + "page_count": 14, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "saxophone", + "area": "violet", + "ad": "Lima", + "doc_count": 4, + "page_count": 36, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "tambourine", + "area": "violet", + "ad": "Lima", + "doc_count": 2, + "page_count": 20, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "timpani", + "area": "orange", + "ad": "Juliet", + "doc_count": 2, + "page_count": 14, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + ], + ) + + # Make sure the view doesn't _crash_ - the template is a dead-simple rendering of the dicts, but this test doesn't prove that + url = urlreverse("ietf.iesg.views.working_groups") + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + class IESGAgendaTests(TestCase): def setUp(self): @@ -759,7 +2343,7 @@ def test_telechat_agenda_content_view(self): urlreverse("ietf.iesg.views.telechat_agenda_content_view", kwargs={"section": section}) ) self.assertContains(r, content, status_code=200) - self.assertEqual(r.get("Content-Type", None), "text/plain") + self.assertEqual(r.get("Content-Type", None), "text/plain; charset=utf-8") def test_telechat_agenda_content_view_permissions(self): for section in TelechatAgendaSectionName.objects.filter(used=True).values_list("slug", flat=True): diff --git a/ietf/iesg/urls.py b/ietf/iesg/urls.py index d8cfec9f90..5fd9dea0cc 100644 --- a/ietf/iesg/urls.py +++ b/ietf/iesg/urls.py @@ -59,6 +59,7 @@ url(r'^agenda/telechat-(?:%(date)s-)?docs.tgz' % settings.URL_REGEXPS, views.telechat_docs_tarfile), url(r'^discusses/$', views.discusses), url(r'^ietf-activity/$', views.ietf_activity), + url(r'^working-groups/$', views.working_groups), url(r'^milestones/$', views.milestones_needing_review), url(r'^photos/$', views.photos), ] diff --git a/ietf/iesg/utils.py b/ietf/iesg/utils.py index 56571dc753..1d24ecac8e 100644 --- a/ietf/iesg/utils.py +++ b/ietf/iesg/utils.py @@ -1,32 +1,45 @@ -from collections import namedtuple +from collections import Counter, defaultdict, namedtuple -import debug # pyflakes:ignore +import datetime + +import debug # pyflakes:ignore + +from django.db import models +from django.utils import timezone from ietf.doc.models import Document, STATUSCHANGE_RELATIONS from ietf.doc.utils_search import fill_in_telechat_date +from ietf.group.models import Group from ietf.iesg.agenda import get_doc_section +from ietf.person.utils import get_active_ads +from ietf.utils.unicodenormalize import normalize_for_sorting +TelechatPageCount = namedtuple( + "TelechatPageCount", + ["for_approval", "for_action", "related", "ad_pages_left_to_ballot_on"], +) -TelechatPageCount = namedtuple('TelechatPageCount',['for_approval','for_action','related','ad_pages_left_to_ballot_on']) def telechat_page_count(date=None, docs=None, ad=None): if not date and not docs: return TelechatPageCount(0, 0, 0, 0) if not docs: - candidates = Document.objects.filter(docevent__telechatdocevent__telechat_date=date).distinct() + candidates = Document.objects.filter( + docevent__telechatdocevent__telechat_date=date + ).distinct() fill_in_telechat_date(candidates) - docs = [ doc for doc in candidates if doc.telechat_date()==date ] + docs = [doc for doc in candidates if doc.telechat_date() == date] - for_action =[d for d in docs if get_doc_section(d).endswith('.3')] + for_action = [d for d in docs if get_doc_section(d).endswith(".3")] - for_approval = set(docs)-set(for_action) + for_approval = set(docs) - set(for_action) - drafts = [d for d in for_approval if d.type_id == 'draft'] + drafts = [d for d in for_approval if d.type_id == "draft"] ad_pages_left_to_ballot_on = 0 pages_for_approval = 0 - + for draft in drafts: pages_for_approval += draft.pages or 0 if ad: @@ -39,30 +52,270 @@ def telechat_page_count(date=None, docs=None, ad=None): pages_for_action = 0 for d in for_action: - if d.type_id == 'draft': + if d.type_id == "draft": pages_for_action += d.pages or 0 - elif d.type_id == 'statchg': + elif d.type_id == "statchg": for rel in d.related_that_doc(STATUSCHANGE_RELATIONS): pages_for_action += rel.pages or 0 - elif d.type_id == 'conflrev': - for rel in d.related_that_doc('conflrev'): + elif d.type_id == "conflrev": + for rel in d.related_that_doc("conflrev"): pages_for_action += rel.pages or 0 else: pass related_pages = 0 - for d in for_approval-set(drafts): - if d.type_id == 'statchg': + for d in for_approval - set(drafts): + if d.type_id == "statchg": for rel in d.related_that_doc(STATUSCHANGE_RELATIONS): related_pages += rel.pages or 0 - elif d.type_id == 'conflrev': - for rel in d.related_that_doc('conflrev'): + elif d.type_id == "conflrev": + for rel in d.related_that_doc("conflrev"): related_pages += rel.pages or 0 else: # There's really nothing to rely on to give a reading load estimate for charters pass - - return TelechatPageCount(for_approval=pages_for_approval, - for_action=pages_for_action, - related=related_pages, - ad_pages_left_to_ballot_on=ad_pages_left_to_ballot_on) + + return TelechatPageCount( + for_approval=pages_for_approval, + for_action=pages_for_action, + related=related_pages, + ad_pages_left_to_ballot_on=ad_pages_left_to_ballot_on, + ) + + +def get_wg_dashboard_info(): + docs = ( + Document.objects.filter( + group__type="wg", + group__state="active", + states__type="draft", + states__slug="active", + ) + .filter(models.Q(ad__isnull=True) | models.Q(ad__in=get_active_ads())) + .distinct() + .prefetch_related("group", "group__parent") + .exclude( + states__type="draft-stream-ietf", + states__slug__in=["c-adopt", "wg-cand", "dead", "parked", "info"], + ) + ) + groups = Group.objects.filter(state="active", type="wg") + areas = Group.objects.filter(state="active", type="area") + + total_group_count = groups.count() + total_doc_count = docs.count() + total_page_count = docs.aggregate(models.Sum("pages"))["pages__sum"] or 0 + totals = { + "group_count": total_group_count, + "doc_count": total_doc_count, + "page_count": total_page_count, + } + + # Since this view is primarily about counting subsets of the above docs query and the + # expected number of returned documents is just under 1000 typically - do the totaling + # work in python rather than asking the db to do it. + + groups_for_area = defaultdict(set) + pages_for_area = defaultdict(lambda: 0) + docs_for_area = defaultdict(lambda: 0) + groups_for_ad = defaultdict(lambda: defaultdict(set)) + pages_for_ad = defaultdict(lambda: defaultdict(lambda: 0)) + docs_for_ad = defaultdict(lambda: defaultdict(lambda: 0)) + groups_for_noad = defaultdict(lambda: defaultdict(set)) + pages_for_noad = defaultdict(lambda: defaultdict(lambda: 0)) + docs_for_noad = defaultdict(lambda: defaultdict(lambda: 0)) + docs_for_wg = defaultdict(lambda: 0) + pages_for_wg = defaultdict(lambda: 0) + groups_total = set() + pages_total = 0 + docs_total = 0 + + responsible_for_group = defaultdict(lambda: defaultdict(lambda: "None")) + responsible_count = defaultdict(lambda: defaultdict(lambda: 0)) + for group in groups: + responsible = f"{', '.join([r.person.plain_name() for r in group.role_set.filter(name_id='ad')])}" + docs_for_noad[responsible][group.parent.acronym] = ( + 0 # Ensure these keys are present later + ) + docs_for_ad[responsible][group.parent.acronym] = 0 + responsible_for_group[group.acronym][group.parent.acronym] = responsible + responsible_count[responsible][group.parent.acronym] += 1 + + for doc in docs: + docs_for_wg[doc.group] += 1 + pages_for_wg[doc.group] += doc.pages + groups_for_area[doc.group.area.acronym].add(doc.group.acronym) + pages_for_area[doc.group.area.acronym] += doc.pages + docs_for_area[doc.group.area.acronym] += 1 + + if doc.ad is None: + responsible = responsible_for_group[doc.group.acronym][ + doc.group.parent.acronym + ] + groups_for_noad[responsible][doc.group.parent.acronym].add( + doc.group.acronym + ) + pages_for_noad[responsible][doc.group.parent.acronym] += doc.pages + docs_for_noad[responsible][doc.group.parent.acronym] += 1 + else: + responsible = f"{doc.ad.plain_name()}" + groups_for_ad[responsible][doc.group.parent.acronym].add(doc.group.acronym) + pages_for_ad[responsible][doc.group.parent.acronym] += doc.pages + docs_for_ad[responsible][doc.group.parent.acronym] += 1 + + docs_total += 1 + groups_total.add(doc.group.acronym) + pages_total += doc.pages + + groups_total = len(groups_total) + totals["groups_with_docs_count"] = groups_total + + area_summary = [] + + for area in areas: + group_count = len(groups_for_area[area.acronym]) + doc_count = docs_for_area[area.acronym] + page_count = pages_for_area[area.acronym] + area_summary.append( + { + "area": area.acronym, + "groups_in_area": groups.filter(parent=area).count(), + "groups_with_docs": group_count, + "doc_count": doc_count, + "page_count": page_count, + "group_percent": group_count / groups_total * 100 + if groups_total != 0 + else 0, + "doc_percent": doc_count / docs_total * 100 if docs_total != 0 else 0, + "page_percent": page_count / pages_total * 100 + if pages_total != 0 + else 0, + } + ) + area_summary.sort(key=lambda r: r["area"]) + area_totals = { + "group_count": groups_total, + "doc_count": docs_total, + "page_count": pages_total, + } + + noad_summary = [] + noad_totals = { + "ad_group_count": 0, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + } + for ad in docs_for_noad: + for area in docs_for_noad[ad]: + noad_totals["ad_group_count"] += responsible_count[ad][area] + noad_totals["doc_group_count"] += len(groups_for_noad[ad][area]) + noad_totals["doc_count"] += docs_for_noad[ad][area] + noad_totals["page_count"] += pages_for_noad[ad][area] + for ad in docs_for_noad: + for area in docs_for_noad[ad]: + noad_summary.append( + { + "ad": ad, + "area": area, + "ad_group_count": responsible_count[ad][area], + "doc_group_count": len(groups_for_noad[ad][area]), + "doc_count": docs_for_noad[ad][area], + "page_count": pages_for_noad[ad][area], + "group_percent": len(groups_for_noad[ad][area]) + / noad_totals["doc_group_count"] + * 100 + if noad_totals["doc_group_count"] != 0 + else 0, + "doc_percent": docs_for_noad[ad][area] + / noad_totals["doc_count"] + * 100 + if noad_totals["doc_count"] != 0 + else 0, + "page_percent": pages_for_noad[ad][area] + / noad_totals["page_count"] + * 100 + if noad_totals["page_count"] != 0 + else 0, + } + ) + noad_summary.sort(key=lambda r: (normalize_for_sorting(r["ad"]), r["area"])) + + ad_summary = [] + ad_totals = { + "ad_group_count": 0, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + } + for ad in docs_for_ad: + for area in docs_for_ad[ad]: + ad_totals["ad_group_count"] += responsible_count[ad][area] + ad_totals["doc_group_count"] += len(groups_for_ad[ad][area]) + ad_totals["doc_count"] += docs_for_ad[ad][area] + ad_totals["page_count"] += pages_for_ad[ad][area] + for ad in docs_for_ad: + for area in docs_for_ad[ad]: + ad_summary.append( + { + "ad": ad, + "area": area, + "ad_group_count": responsible_count[ad][area], + "doc_group_count": len(groups_for_ad[ad][area]), + "doc_count": docs_for_ad[ad][area], + "page_count": pages_for_ad[ad][area], + "group_percent": len(groups_for_ad[ad][area]) + / ad_totals["doc_group_count"] + * 100 + if ad_totals["doc_group_count"] != 0 + else 0, + "doc_percent": docs_for_ad[ad][area] / ad_totals["doc_count"] * 100 + if ad_totals["doc_count"] != 0 + else 0, + "page_percent": pages_for_ad[ad][area] + / ad_totals["page_count"] + * 100 + if ad_totals["page_count"] != 0 + else 0, + } + ) + ad_summary.sort(key=lambda r: (normalize_for_sorting(r["ad"]), r["area"])) + + rfc_counter = Counter( + Document.objects.filter(type="rfc").values_list("group__acronym", flat=True) + ) + recent_rfc_counter = Counter( + Document.objects.filter( + type="rfc", + docevent__type="published_rfc", + docevent__time__gte=timezone.now() - datetime.timedelta(weeks=104), + ).values_list("group__acronym", flat=True) + ) + for wg in set(groups) - set(docs_for_wg.keys()): + docs_for_wg[wg] += 0 + pages_for_wg[wg] += 0 + wg_summary = [] + for wg in docs_for_wg: + wg_summary.append( + { + "wg": wg.acronym, + "area": wg.parent.acronym, + "ad": responsible_for_group[wg.acronym][wg.parent.acronym], + "doc_count": docs_for_wg[wg], + "page_count": pages_for_wg[wg], + "rfc_count": rfc_counter[wg.acronym], + "recent_rfc_count": recent_rfc_counter[wg.acronym], + } + ) + wg_summary.sort(key=lambda r: (r["wg"], r["area"])) + + return ( + area_summary, + area_totals, + ad_summary, + noad_summary, + ad_totals, + noad_totals, + totals, + wg_summary, + ) diff --git a/ietf/iesg/views.py b/ietf/iesg/views.py index df02754f2e..f03afb9fc1 100644 --- a/ietf/iesg/views.py +++ b/ietf/iesg/views.py @@ -61,13 +61,14 @@ from ietf.group.models import GroupMilestone, Role from ietf.iesg.agenda import agenda_data, agenda_sections, fill_in_agenda_docs, get_agenda_date from ietf.iesg.models import TelechatDate, TelechatAgendaContent -from ietf.iesg.utils import telechat_page_count +from ietf.iesg.utils import get_wg_dashboard_info, telechat_page_count from ietf.ietfauth.utils import has_role, role_required, user_is_person from ietf.name.models import TelechatAgendaSectionName from ietf.person.models import Person from ietf.meeting.utils import get_activity_stats from ietf.doc.utils_search import fill_in_document_table_attributes, fill_in_telechat_date from ietf.utils.timezone import date_today, datetime_from_date +from ietf.utils.unicodenormalize import normalize_for_sorting def review_decisions(request, year=None): events = DocEvent.objects.filter(type__in=("iesg_disapproved", "iesg_approved")) @@ -101,7 +102,7 @@ def agenda_json(request, date=None): res = { "telechat-date": str(data["date"]), - "as-of": str(datetime.datetime.utcnow()), + "as-of": str(datetime.datetime.now(datetime.UTC)), "page-counts": telechat_page_count(date=get_agenda_date(date))._asdict(), "sections": {}, } @@ -221,7 +222,7 @@ def agenda_txt(request, date=None): "date": data["date"], "sections": sorted(data["sections"].items(), key=lambda x:[int(p) for p in x[0].split('.')]), "domain": Site.objects.get_current().domain, - }, content_type="text/plain; charset=%s"%settings.DEFAULT_CHARSET) + }, content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}") @role_required('Area Director', 'Secretariat') def agenda_moderator_package(request, date=None): @@ -277,14 +278,23 @@ def leaf_section(num, section): @role_required('Area Director', 'Secretariat') def agenda_package(request, date=None): data = agenda_data(date) - return render(request, "iesg/agenda_package.txt", { + return render( + request, + "iesg/agenda_package.txt", + { "date": data["date"], "sections": sorted(data["sections"].items()), "roll_call": data["sections"]["1.1"]["text"], "minutes": data["sections"]["1.3"]["text"], - "management_items": [(num, section) for num, section in data["sections"].items() if "6" < num < "7"], + "management_items": [ + (num, section) + for num, section in data["sections"].items() + if "6" < num < "7" + ], "domain": Site.objects.get_current().domain, - }, content_type='text/plain') + }, + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + ) def agenda_documents_txt(request): @@ -315,7 +325,10 @@ def agenda_documents_txt(request): d.rev, ) rows.append("\t".join(row)) - return HttpResponse("\n".join(rows), content_type='text/plain') + return HttpResponse( + "\n".join(rows), + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + ) class RescheduleForm(forms.Form): telechat_date = forms.TypedChoiceField(coerce=lambda x: datetime.datetime.strptime(x, '%Y-%m-%d').date(), empty_value=None, required=False) @@ -535,7 +548,7 @@ def milestones_needing_review(request): ) return render(request, 'iesg/milestones_needing_review.html', - dict(ads=sorted(ad_list, key=lambda ad: ad.plain_name()),)) + dict(ads=sorted(ad_list, key=lambda ad: normalize_for_sorting(ad.plain_name())),)) def photos(request): roles = sorted(Role.objects.filter(group__type='area', group__state='active', name_id='ad'),key=lambda x: "" if x.group.acronym=="gen" else x.group.acronym) @@ -610,4 +623,17 @@ def telechat_agenda_content_manage(request): @role_required("Secretariat", "IAB Chair", "Area Director") def telechat_agenda_content_view(request, section): content = get_object_or_404(TelechatAgendaContent, section__slug=section, section__used=True) - return HttpResponse(content=content.text, content_type="text/plain") + return HttpResponse( + content=content.text, + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + ) + +def working_groups(request): + + area_summary, area_totals, ad_summary, noad_summary, ad_totals, noad_totals, totals, wg_summary = get_wg_dashboard_info() + + return render( + request, + "iesg/working_groups.html", + dict(area_summary=area_summary, area_totals=area_totals, ad_summary=ad_summary, noad_summary=noad_summary, ad_totals=ad_totals, noad_totals=noad_totals, totals=totals, wg_summary=wg_summary), + ) diff --git a/ietf/ietfauth/admin.py b/ietf/ietfauth/admin.py new file mode 100644 index 0000000000..c2914f9efa --- /dev/null +++ b/ietf/ietfauth/admin.py @@ -0,0 +1,136 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +import datetime + +from django.conf import settings +from django.contrib import admin, messages +from django.contrib.admin import action +from django.contrib.admin.actions import delete_selected as default_delete_selected +from django.contrib.auth.admin import UserAdmin +from django.contrib.auth.models import User +from django.utils import timezone + + +# Replace default UserAdmin with our custom one +admin.site.unregister(User) + + +class AgeListFilter(admin.SimpleListFilter): + title = "account age" + parameter_name = "age" + + def lookups(self, request, model_admin): + return [ + ("1day", "> 1 day"), + ("3days", "> 3 days"), + ("1week", "> 1 week"), + ("1month", "> 1 month"), + ("1year", "> 1 year"), + ] + + def queryset(self, request, queryset): + deltas = { + "1day": datetime.timedelta(days=1), + "3days": datetime.timedelta(days=3), + "1week": datetime.timedelta(weeks=1), + "1month": datetime.timedelta(days=30), + "1year": datetime.timedelta(days=365), + } + if self.value(): + return queryset.filter(date_joined__lt=timezone.now()-deltas[self.value()]) + return queryset + + +@admin.register(User) +class CustomUserAdmin(UserAdmin): + list_display = ( + "username", + "person", + "date_joined", + "last_login", + "is_staff", + ) + list_filter = list(UserAdmin.list_filter) + [ + AgeListFilter, + ("person", admin.EmptyFieldListFilter), + ] + actions = ["delete_selected"] + + @action( + permissions=["delete"], description="Delete personless %(verbose_name_plural)s" + ) + def delete_selected(self, request, queryset): + """Delete selected action restricted to Users with a null Person field + + This displaces the default delete_selected action with a safer one that will + only delete personless Users. It is done this way instead of by introducing + a new action so that we can simply hand off to the default action (imported + as default_delete_selected()) without having to adjust its template (and maybe + other things) to make it work with a different action name. + """ + already_confirmed = bool(request.POST.get("post")) + personless_queryset = queryset.filter(person__isnull=True) + original_count = queryset.count() + personless_count = personless_queryset.count() + if personless_count > original_count: + # Refuse to act if the count increased! + self.message_user( + request, + ( + "Limiting the selection to Users without a Person INCREASED the " + "count from {} to {}. This should not happen and probably means a " + "concurrent change to the database affected this request. Please " + "try again.".format(original_count, personless_count) + ), + level=messages.ERROR, + ) + return None # return to changelist + + # Display warning/info if this is showing the confirmation page + if not already_confirmed: + if personless_count < original_count: + self.message_user( + request, + ( + "Limiting the selection to Users without a Person reduced the " + "count from {} to {}. Only {} will be deleted.".format( + original_count, personless_count, personless_count + ) + ), + level=messages.WARNING, + ) + else: + self.message_user( + request, + "Confirmed that all selected Users had no Persons.", + ) + + # Django limits the number of fields in a request. The delete form itself + # includes a few metadata fields, so give it a little padding. The default + # limit is 1000 and everything will break if it's a small number, so not + # bothering to check that it's > 10. + max_count = settings.DATA_UPLOAD_MAX_NUMBER_FIELDS - 10 + if personless_count > max_count: + self.message_user( + request, + ( + f"Only {max_count} Users can be deleted at once. Will only delete " + f"the first {max_count} selected Personless Users." + ), + level=messages.WARNING, + ) + # delete() doesn't like a queryset limited via [:max_count], so do an + # equivalent filter. + last_to_delete = personless_queryset.order_by("pk")[max_count] + personless_queryset = personless_queryset.filter(pk__lt=last_to_delete.pk) + + if already_confirmed and personless_count != original_count: + # After confirmation, none of the above filtering should change anything. + # Refuse to delete if the DB moved underneath us. + self.message_user( + request, + "Queryset count changed, nothing deleted. Please try again.", + level=messages.ERROR, + ) + return None + + return default_delete_selected(self, request, personless_queryset) diff --git a/ietf/ietfauth/forms.py b/ietf/ietfauth/forms.py index a70f7b6ca1..41828f2bf6 100644 --- a/ietf/ietfauth/forms.py +++ b/ietf/ietfauth/forms.py @@ -3,16 +3,19 @@ import re + from unidecode import unidecode from django import forms +from django.contrib.auth.models import User +from django.contrib.auth import password_validation from django.core.exceptions import ValidationError from django.db import models -from django.contrib.auth.models import User from ietf.person.models import Person, Email from ietf.mailinglists.models import Allowlisted from ietf.utils.text import isascii +from .password_validation import StrongPasswordValidator from .validators import prevent_at_symbol, prevent_system_name, prevent_anonymous_name, is_allowed_address from .widgets import PasswordStrengthInput, PasswordConfirmationInput @@ -30,21 +33,56 @@ def clean_email(self): return email +class PasswordStrengthField(forms.CharField): + widget = PasswordStrengthInput( + attrs={ + "class": "password_strength", + "data-disable-strength-enforcement": "", # usually removed in init + } + ) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + for pwval in password_validation.get_default_password_validators(): + if isinstance(pwval, password_validation.MinimumLengthValidator): + self.widget.attrs["minlength"] = pwval.min_length + elif isinstance(pwval, StrongPasswordValidator): + self.widget.attrs.pop( + "data-disable-strength-enforcement", None + ) + + + class PasswordForm(forms.Form): - password = forms.CharField(widget=PasswordStrengthInput(attrs={'class':'password_strength'})) + password = PasswordStrengthField() password_confirmation = forms.CharField(widget=PasswordConfirmationInput( confirm_with='password', attrs={'class':'password_confirmation'}), help_text="Enter the same password as above, for verification.",) - + + def __init__(self, *args, user=None, **kwargs): + # user is a kw-only argument to avoid interfering with the signature + # when this class is mixed with ModelForm in PersonPasswordForm + self.user = user + super().__init__(*args, **kwargs) def clean_password_confirmation(self): - password = self.cleaned_data.get("password", "") - password_confirmation = self.cleaned_data["password_confirmation"] + # clean fields here rather than a clean() method so validation is + # still enforced in PersonPasswordForm without having to override its + # clean() method + password = self.cleaned_data.get("password") + password_confirmation = self.cleaned_data.get("password_confirmation") if password != password_confirmation: - raise forms.ValidationError("The two password fields didn't match.") + raise ValidationError( + "The password confirmation is different than the new password" + ) + try: + password_validation.validate_password(password_confirmation, self.user) + except ValidationError as err: + self.add_error("password", err) return password_confirmation + def ascii_cleaner(supposedly_ascii): outside_printable_ascii_pattern = r'[^\x20-\x7F]' if re.search(outside_printable_ascii_pattern, supposedly_ascii): @@ -170,33 +208,21 @@ class Meta: model = Allowlisted exclude = ['by', 'time' ] - -from django import forms - -class ChangePasswordForm(forms.Form): +class ChangePasswordForm(PasswordForm): current_password = forms.CharField(widget=forms.PasswordInput) + field_order = ["current_password", "password", "password_confirmation"] - new_password = forms.CharField(widget=PasswordStrengthInput(attrs={'class':'password_strength'})) - new_password_confirmation = forms.CharField(widget=PasswordConfirmationInput( - confirm_with='new_password', - attrs={'class':'password_confirmation'})) - - def __init__(self, user, data=None): - self.user = user - super(ChangePasswordForm, self).__init__(data) + def __init__(self, user, *args, **kwargs): + # user arg is optional in superclass, but required for this form + super().__init__(*args, user=user, **kwargs) def clean_current_password(self): - password = self.cleaned_data.get('current_password', None) + # n.b., password = None is handled by check_password and results in a failed check + password = self.cleaned_data.get("current_password", None) if not self.user.check_password(password): - raise ValidationError('Invalid password') + raise ValidationError("Invalid password") return password - - def clean(self): - new_password = self.cleaned_data.get('new_password', None) - conf_password = self.cleaned_data.get('new_password_confirmation', None) - if not new_password == conf_password: - raise ValidationError("The password confirmation is different than the new password") class ChangeUsernameForm(forms.Form): diff --git a/ietf/ietfauth/password_validation.py b/ietf/ietfauth/password_validation.py new file mode 100644 index 0000000000..bfed4a784e --- /dev/null +++ b/ietf/ietfauth/password_validation.py @@ -0,0 +1,23 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +from django.core.exceptions import ValidationError +from zxcvbn import zxcvbn + + +class StrongPasswordValidator: + message = "This password does not meet complexity requirements and is easily guessable." + code = "weak" + min_zxcvbn_score = 3 + + def __init__(self, message=None, code=None, min_zxcvbn_score=None): + if message is not None: + self.message = message + if code is not None: + self.code = code + if min_zxcvbn_score is not None: + self.min_zxcvbn_score = min_zxcvbn_score + + def validate(self, password, user=None): + """Validate that a password is strong enough""" + strength_report = zxcvbn(password[:72], max_length=72) + if strength_report["score"] < self.min_zxcvbn_score: + raise ValidationError(message=self.message, code=self.code) diff --git a/ietf/ietfauth/tests.py b/ietf/ietfauth/tests.py index fed764e8bd..a77e5bd5d5 100644 --- a/ietf/ietfauth/tests.py +++ b/ietf/ietfauth/tests.py @@ -32,14 +32,13 @@ from ietf.group.factories import GroupFactory, RoleFactory from ietf.group.models import Group, Role, RoleName from ietf.ietfauth.utils import has_role -from ietf.meeting.factories import MeetingFactory +from ietf.meeting.factories import MeetingFactory, RegistrationFactory, RegistrationTicketFactory from ietf.nomcom.factories import NomComFactory from ietf.person.factories import PersonFactory, EmailFactory, UserFactory, PersonalApiKeyFactory from ietf.person.models import Person, Email from ietf.person.tasks import send_apikey_usage_emails_task from ietf.review.factories import ReviewRequestFactory, ReviewAssignmentFactory from ietf.review.models import ReviewWish, UnavailablePeriod -from ietf.stats.models import MeetingRegistration from ietf.utils.mail import outbox, empty_outbox, get_payload_text from ietf.utils.test_utils import TestCase, login_testing_unauthorized from ietf.utils.timezone import date_today @@ -169,18 +168,40 @@ def register_and_verify(self, email): self.assertEqual(r.status_code, 200) # password mismatch - r = self.client.post(confirm_url, { 'password': 'secret', 'password_confirmation': 'nosecret' }) + r = self.client.post( + confirm_url, { + "password": "secret-and-secure", + "password_confirmation": "not-secret-or-secure", + } + ) + self.assertEqual(r.status_code, 200) + self.assertEqual(User.objects.filter(username=email).count(), 0) + + # weak password + r = self.client.post( + confirm_url, { + "password": "password1234", + "password_confirmation": "password1234", + } + ) self.assertEqual(r.status_code, 200) self.assertEqual(User.objects.filter(username=email).count(), 0) # confirm - r = self.client.post(confirm_url, { 'name': 'User Name', 'ascii': 'User Name', 'password': 'secret', 'password_confirmation': 'secret' }) + r = self.client.post( + confirm_url, + { + "name": "User Name", + "ascii": "User Name", + "password": "secret-and-secure", + "password_confirmation": "secret-and-secure", + }, + ) self.assertEqual(r.status_code, 200) self.assertEqual(User.objects.filter(username=email).count(), 1) self.assertEqual(Person.objects.filter(user__username=email).count(), 1) self.assertEqual(Email.objects.filter(person__user__username=email).count(), 1) - # This also tests new account creation. def test_create_existing_account(self): # create account once @@ -393,14 +414,15 @@ def test_nomcom_dressing_on_profile(self): self.assertFalse(q('#volunteer-button')) self.assertTrue(q('#volunteered')) - def test_reset_password(self): + WEAK_PASSWORD="password1234" + VALID_PASSWORD = "complex-and-long-valid-password" + ANOTHER_VALID_PASSWORD = "very-complicated-and-lengthy-password" url = urlreverse("ietf.ietfauth.views.password_reset") - email = 'someone@example.com' - password = 'foobar' + email = "someone@example.com" user = PersonFactory(user__email=email).user - user.set_password(password) + user.set_password(VALID_PASSWORD) user.save() # get @@ -408,21 +430,23 @@ def test_reset_password(self): self.assertEqual(r.status_code, 200) # ask for reset, wrong username (form should not fail) - r = self.client.post(url, { 'username': "nobody@example.com" }) + r = self.client.post(url, {"username": "nobody@example.com"}) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertTrue(len(q("form .is-invalid")) == 0) # ask for reset empty_outbox() - r = self.client.post(url, { 'username': user.username }) + r = self.client.post(url, {"username": user.username}) self.assertEqual(r.status_code, 200) self.assertEqual(len(outbox), 1) # goto change password page, logged in as someone else confirm_url = self.extract_confirm_url(outbox[-1]) other_user = UserFactory() - self.client.login(username=other_user.username, password=other_user.username + '+password') + self.client.login( + username=other_user.username, password=other_user.username + "+password" + ) r = self.client.get(confirm_url) self.assertEqual(r.status_code, 403) @@ -431,17 +455,44 @@ def test_reset_password(self): r = self.client.get(confirm_url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertNotIn(user.username, q('.nav').text(), - 'user should not appear signed in while resetting password') + self.assertNotIn( + user.username, + q(".nav").text(), + "user should not appear signed in while resetting password", + ) # password mismatch - r = self.client.post(confirm_url, { 'password': 'secret', 'password_confirmation': 'nosecret' }) + r = self.client.post( + confirm_url, + { + "password": ANOTHER_VALID_PASSWORD, + "password_confirmation": ANOTHER_VALID_PASSWORD[::-1], + }, + ) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + self.assertTrue(len(q("form .is-invalid")) > 0) + + # weak password + r = self.client.post( + confirm_url, + { + "password": WEAK_PASSWORD, + "password_confirmation": WEAK_PASSWORD, + }, + ) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertTrue(len(q("form .is-invalid")) > 0) # confirm - r = self.client.post(confirm_url, { 'password': 'secret', 'password_confirmation': 'secret' }) + r = self.client.post( + confirm_url, + { + "password": ANOTHER_VALID_PASSWORD, + "password_confirmation": ANOTHER_VALID_PASSWORD, + }, + ) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertEqual(len(q("form .is-invalid")), 0) @@ -452,15 +503,18 @@ def test_reset_password(self): # login after reset request empty_outbox() - user.set_password(password) + user.set_password(VALID_PASSWORD) user.save() - r = self.client.post(url, { 'username': user.username }) + r = self.client.post(url, {"username": user.username}) self.assertEqual(r.status_code, 200) self.assertEqual(len(outbox), 1) confirm_url = self.extract_confirm_url(outbox[-1]) - r = self.client.post(urlreverse("ietf.ietfauth.views.login"), {'username': email, 'password': password}) + r = self.client.post( + urlreverse("ietf.ietfauth.views.login"), + {"username": email, "password": VALID_PASSWORD}, + ) r = self.client.get(confirm_url) self.assertEqual(r.status_code, 404) @@ -468,12 +522,12 @@ def test_reset_password(self): # change password after reset request empty_outbox() - r = self.client.post(url, { 'username': user.username }) + r = self.client.post(url, {"username": user.username}) self.assertEqual(r.status_code, 200) self.assertEqual(len(outbox), 1) confirm_url = self.extract_confirm_url(outbox[-1]) - user.set_password('newpassword') + user.set_password(ANOTHER_VALID_PASSWORD) user.save() r = self.client.get(confirm_url) @@ -587,98 +641,175 @@ def test_review_overview(self): self.assertEqual(ReviewWish.objects.filter(doc=doc, team=review_req.team).count(), 0) def test_change_password(self): + VALID_PASSWORD = "complex-and-long-valid-password" + ANOTHER_VALID_PASSWORD = "very-complicated-and-lengthy-password" chpw_url = urlreverse("ietf.ietfauth.views.change_password") prof_url = urlreverse("ietf.ietfauth.views.profile") login_url = urlreverse("ietf.ietfauth.views.login") - redir_url = '%s?next=%s' % (login_url, chpw_url) + redir_url = "%s?next=%s" % (login_url, chpw_url) # get without logging in r = self.client.get(chpw_url) self.assertRedirects(r, redir_url) - user = User.objects.create(username="someone@example.com", email="someone@example.com") - user.set_password("password") + user = User.objects.create( + username="someone@example.com", email="someone@example.com" + ) + user.set_password(VALID_PASSWORD) user.save() p = Person.objects.create(name="Some One", ascii="Some One", user=user) Email.objects.create(address=user.username, person=p, origin=user.username) # log in - r = self.client.post(redir_url, {"username":user.username, "password":"password"}) + r = self.client.post( + redir_url, {"username": user.username, "password": VALID_PASSWORD} + ) self.assertRedirects(r, chpw_url) # wrong current password - r = self.client.post(chpw_url, {"current_password": "fiddlesticks", - "new_password": "foobar", - "new_password_confirmation": "foobar", - }) + r = self.client.post( + chpw_url, + { + "current_password": "fiddlesticks", + "password": ANOTHER_VALID_PASSWORD, + "password_confirmation": ANOTHER_VALID_PASSWORD, + }, + ) self.assertEqual(r.status_code, 200) - self.assertFormError(r.context["form"], 'current_password', 'Invalid password') + self.assertFormError(r.context["form"], "current_password", "Invalid password") # mismatching new passwords - r = self.client.post(chpw_url, {"current_password": "password", - "new_password": "foobar", - "new_password_confirmation": "barfoo", - }) + r = self.client.post( + chpw_url, + { + "current_password": VALID_PASSWORD, + "password": ANOTHER_VALID_PASSWORD, + "password_confirmation": ANOTHER_VALID_PASSWORD[::-1], + }, + ) self.assertEqual(r.status_code, 200) - self.assertFormError(r.context["form"], None, "The password confirmation is different than the new password") + self.assertFormError( + r.context["form"], + "password_confirmation", + "The password confirmation is different than the new password", + ) + + # password too short + r = self.client.post( + chpw_url, + { + "current_password": VALID_PASSWORD, + "password": "sh0rtpw0rd", + "password_confirmation": "sh0rtpw0rd", + } + ) + self.assertEqual(r.status_code, 200) + self.assertFormError( + r.context["form"], + "password", + "This password is too short. It must contain at least " + f"{settings.PASSWORD_POLICY_MIN_LENGTH} characters." + ) + + # password too simple + r = self.client.post( + chpw_url, + { + "current_password": VALID_PASSWORD, + "password": "passwordpassword", + "password_confirmation": "passwordpassword", + } + ) + self.assertEqual(r.status_code, 200) + self.assertFormError( + r.context["form"], + "password", + "This password does not meet complexity requirements " + "and is easily guessable." + ) # correct password change - r = self.client.post(chpw_url, {"current_password": "password", - "new_password": "foobar", - "new_password_confirmation": "foobar", - }) + r = self.client.post( + chpw_url, + { + "current_password": VALID_PASSWORD, + "password": ANOTHER_VALID_PASSWORD, + "password_confirmation": ANOTHER_VALID_PASSWORD, + }, + ) self.assertRedirects(r, prof_url) # refresh user object user = User.objects.get(username="someone@example.com") - self.assertTrue(user.check_password('foobar')) + self.assertTrue(user.check_password(ANOTHER_VALID_PASSWORD)) def test_change_username(self): - + VALID_PASSWORD = "complex-and-long-valid-password" chun_url = urlreverse("ietf.ietfauth.views.change_username") prof_url = urlreverse("ietf.ietfauth.views.profile") login_url = urlreverse("ietf.ietfauth.views.login") - redir_url = '%s?next=%s' % (login_url, chun_url) + redir_url = "%s?next=%s" % (login_url, chun_url) # get without logging in r = self.client.get(chun_url) self.assertRedirects(r, redir_url) - user = User.objects.create(username="someone@example.com", email="someone@example.com") - user.set_password("password") + user = User.objects.create( + username="someone@example.com", email="someone@example.com" + ) + user.set_password(VALID_PASSWORD) user.save() p = Person.objects.create(name="Some One", ascii="Some One", user=user) Email.objects.create(address=user.username, person=p, origin=user.username) - Email.objects.create(address="othername@example.org", person=p, origin=user.username) + Email.objects.create( + address="othername@example.org", person=p, origin=user.username + ) # log in - r = self.client.post(redir_url, {"username":user.username, "password":"password"}) + r = self.client.post( + redir_url, {"username": user.username, "password": VALID_PASSWORD} + ) self.assertRedirects(r, chun_url) # wrong username - r = self.client.post(chun_url, {"username": "fiddlesticks", - "password": "password", - }) + r = self.client.post( + chun_url, + { + "username": "fiddlesticks", + "password": VALID_PASSWORD, + }, + ) self.assertEqual(r.status_code, 200) - self.assertFormError(r.context["form"], 'username', - "Select a valid choice. fiddlesticks is not one of the available choices.") + self.assertFormError( + r.context["form"], + "username", + "Select a valid choice. fiddlesticks is not one of the available choices.", + ) # wrong password - r = self.client.post(chun_url, {"username": "othername@example.org", - "password": "foobar", - }) + r = self.client.post( + chun_url, + { + "username": "othername@example.org", + "password": "foobar", + }, + ) self.assertEqual(r.status_code, 200) - self.assertFormError(r.context["form"], 'password', 'Invalid password') + self.assertFormError(r.context["form"], "password", "Invalid password") # correct username change - r = self.client.post(chun_url, {"username": "othername@example.org", - "password": "password", - }) + r = self.client.post( + chun_url, + { + "username": "othername@example.org", + "password": VALID_PASSWORD, + }, + ) self.assertRedirects(r, prof_url) # refresh user object prev = user user = User.objects.get(username="othername@example.org") self.assertEqual(prev, user) - self.assertTrue(user.check_password('password')) + self.assertTrue(user.check_password(VALID_PASSWORD)) def test_apikey_management(self): # Create a person with a role that will give at least one valid apikey @@ -1016,11 +1147,15 @@ def test_oidc_code_auth(self): EmailFactory(person=person) email_list = person.email_set.all().values_list('address', flat=True) meeting = MeetingFactory(type_id='ietf', date=date_today()) - MeetingRegistration.objects.create( - meeting=meeting, person=None, first_name=person.first_name(), last_name=person.last_name(), - email=email_list[0], ticket_type='full_week', reg_type='remote', affiliation='Some Company', - ) - + reg_person = RegistrationFactory( + meeting=meeting, + person=person, + first_name=person.first_name(), + last_name=person.last_name(), + email=email_list[0], + affiliation='Some Company', + with_ticket={'attendance_type_id': 'remote', 'ticket_type_id': 'week_pass'}, + ) # Get access authorisation session = {} session["state"] = rndstr() @@ -1073,35 +1208,48 @@ def test_oidc_code_auth(self): for key in ['iss', 'sub', 'aud', 'exp', 'iat', 'auth_time', 'nonce', 'at_hash']: self.assertIn(key, access_token_info['id_token']) - # Get userinfo, check keys present + # Get userinfo, check keys present, most common scenario userinfo = client.do_user_info_request(state=params["state"], scope=args['scope']) for key in [ 'email', 'family_name', 'given_name', 'meeting', 'name', 'pronouns', 'roles', 'ticket_type', 'reg_type', 'affiliation', 'picture', 'dots', ]: self.assertIn(key, userinfo) self.assertTrue(userinfo[key]) self.assertIn('remote', set(userinfo['reg_type'].split())) - self.assertNotIn('hackathon', set(userinfo['reg_type'].split())) + self.assertNotIn('hackathon_onsite', set(userinfo['reg_type'].split())) self.assertIn(active_group.acronym, [i[1] for i in userinfo['roles']]) self.assertNotIn(closed_group.acronym, [i[1] for i in userinfo['roles']]) - # Create another registration, with a different email - MeetingRegistration.objects.create( - meeting=meeting, person=None, first_name=person.first_name(), last_name=person.last_name(), - email=email_list[1], ticket_type='one_day', reg_type='hackathon', affiliation='Some Company, Inc', - ) + # Create a registration, with only email, no person (rare if at all) + reg_person.delete() + reg_email = RegistrationFactory( + meeting=meeting, + person=None, + first_name=person.first_name(), + last_name=person.last_name(), + email=email_list[1], + affiliation='Some Company, Inc', + with_ticket={'attendance_type_id': 'hackathon_onsite', 'ticket_type_id': 'one_day'}, + ) userinfo = client.do_user_info_request(state=params["state"], scope=args['scope']) - self.assertIn('hackathon', set(userinfo['reg_type'].split())) - self.assertIn('remote', set(userinfo['reg_type'].split())) - self.assertIn('full_week', set(userinfo['ticket_type'].split())) - self.assertIn('Some Company', userinfo['affiliation']) - - # Create a third registration, with a composite reg type - MeetingRegistration.objects.create( - meeting=meeting, person=None, first_name=person.first_name(), last_name=person.last_name(), - email=email_list[1], ticket_type='one_day', reg_type='hackathon remote', affiliation='Some Company, Inc', - ) + self.assertIn('hackathon_onsite', set(userinfo['reg_type'].split())) + self.assertNotIn('remote', set(userinfo['reg_type'].split())) + self.assertIn('one_day', set(userinfo['ticket_type'].split())) + self.assertIn('Some Company, Inc', userinfo['affiliation']) + + # Test with multiple tickets + reg_email.delete() + creg = RegistrationFactory( + meeting=meeting, + person=None, + first_name=person.first_name(), + last_name=person.last_name(), + email=email_list[1], + affiliation='Some Company, Inc', + with_ticket={'attendance_type_id': 'hackathon_remote', 'ticket_type_id': 'week_pass'}, + ) + RegistrationTicketFactory(registration=creg, attendance_type_id='remote', ticket_type_id='week_pass') userinfo = client.do_user_info_request(state=params["state"], scope=args['scope']) - self.assertEqual(set(userinfo['reg_type'].split()), set(['remote', 'hackathon'])) + self.assertEqual(set(userinfo['reg_type'].split()), set(['remote', 'hackathon_remote'])) # Check that ending a session works r = client.do_end_session_request(state=params["state"], scope=args['scope']) diff --git a/ietf/ietfauth/utils.py b/ietf/ietfauth/utils.py index b4c6da14ea..0df667fbd2 100644 --- a/ietf/ietfauth/utils.py +++ b/ietf/ietfauth/utils.py @@ -137,6 +137,10 @@ def has_role(user, role_names, *args, **kwargs): group__type="sdo", group__state="active", ), + "Liaison Coordinator": Q( + name="liaison_coordinator", + group__acronym="iab", + ), "Authorized Individual": Q( name="auth", group__type="sdo", @@ -207,9 +211,9 @@ def role_required(*role_names): # specific permissions + def is_authorized_in_doc_stream(user, doc): - """Return whether user is authorized to perform stream duties on - document.""" + """Is user authorized to perform stream duties on doc?""" if has_role(user, ["Secretariat"]): return True @@ -283,7 +287,7 @@ def is_individual_draft_author(user, doc): if not hasattr(user, 'person'): return False - if user.person in doc.authors(): + if user.person in doc.author_persons(): return True return False @@ -346,13 +350,14 @@ def scope_pronouns(self): ) def scope_registration(self): + # import here to avoid circular imports from ietf.meeting.helpers import get_current_ietf_meeting - from ietf.stats.models import MeetingRegistration + from ietf.meeting.models import Registration meeting = get_current_ietf_meeting() person = self.user.person email_list = person.email_set.values_list('address') q = Q(person=person, meeting=meeting) | Q(email__in=email_list, meeting=meeting) - regs = MeetingRegistration.objects.filter(q).distinct() + regs = Registration.objects.filter(q).distinct() for reg in regs: if not reg.person_id: reg.person = person @@ -363,19 +368,20 @@ def scope_registration(self): ticket_types = set([]) reg_types = set([]) for reg in regs: - ticket_types.add(reg.ticket_type) - reg_types.add(reg.reg_type) + for ticket in reg.tickets.all(): + ticket_types.add(ticket.ticket_type.slug) + reg_types.add(ticket.attendance_type.slug) info = { - 'meeting': meeting.number, + 'meeting': meeting.number, # full_week, one_day, student: - 'ticket_type': ' '.join(ticket_types), + 'ticket_type': ' '.join(ticket_types), # onsite, remote, hackathon_onsite, hackathon_remote: - 'reg_type': ' '.join(reg_types), - 'affiliation': ([ reg.affiliation for reg in regs if reg.affiliation ] or [''])[0], + 'reg_type': ' '.join(reg_types), + 'affiliation': ([reg.affiliation for reg in regs if reg.affiliation] or [''])[0], } return info - + def can_request_rfc_publication(user, doc): """Answers whether this user has an appropriate role to send this document to the RFC Editor for publication as an RFC. diff --git a/ietf/ietfauth/views.py b/ietf/ietfauth/views.py index 23f66ce824..b5256b14f8 100644 --- a/ietf/ietfauth/views.py +++ b/ietf/ietfauth/views.py @@ -38,14 +38,14 @@ import importlib # needed if we revert to higher barrier for account creation -#from datetime import datetime as DateTime, timedelta as TimeDelta, date as Date +# from datetime import datetime as DateTime, timedelta as TimeDelta, date as Date from collections import defaultdict import django.core.signing from django import forms from django.contrib import messages from django.conf import settings -from django.contrib.auth import logout, update_session_auth_hash +from django.contrib.auth import logout, update_session_auth_hash, password_validation from django.contrib.auth.decorators import login_required from django.contrib.auth.forms import AuthenticationForm from django.contrib.auth.hashers import identify_hasher @@ -80,7 +80,6 @@ # These are needed if we revert to the higher bar for account creation - def index(request): return render(request, 'registration/index.html') @@ -97,7 +96,7 @@ def index(request): # def ietf_login(request): # if not request.user.is_authenticated: # return HttpResponse("Not authenticated?", status=500) -# +# # redirect_to = request.REQUEST.get(REDIRECT_FIELD_NAME, '') # request.session.set_test_cookie() # return HttpResponseRedirect('/accounts/loggedin/?%s=%s' % (REDIRECT_FIELD_NAME, urlquote(redirect_to))) @@ -518,7 +517,7 @@ def confirm_password_reset(request, auth): password = data['password'] last_login = None if data['last_login']: - last_login = datetime.datetime.fromtimestamp(data['last_login'], datetime.timezone.utc) + last_login = datetime.datetime.fromtimestamp(data['last_login'], datetime.UTC) except django.core.signing.BadSignature: raise Http404("Invalid or expired auth") @@ -530,7 +529,7 @@ def confirm_password_reset(request, auth): ) success = False if request.method == 'POST': - form = PasswordForm(request.POST) + form = PasswordForm(user=user, data=request.POST) if form.is_valid(): password = form.cleaned_data["password"] @@ -539,7 +538,7 @@ def confirm_password_reset(request, auth): success = True else: - form = PasswordForm() + form = PasswordForm(user=user) hlibname, hashername = settings.PASSWORD_HASHERS[0].rsplit('.',1) hlib = importlib.import_module(hlibname) @@ -582,7 +581,6 @@ def test_email(request): return r - class AddReviewWishForm(forms.Form): doc = SearchableDocumentField(label="Document", doc_type="draft") team = forms.ModelChoiceField(queryset=Group.objects.all(), empty_label="(Choose review team)") @@ -671,7 +669,7 @@ def change_password(request): if request.method == 'POST': form = ChangePasswordForm(user, request.POST) if form.is_valid(): - new_password = form.cleaned_data["new_password"] + new_password = form.cleaned_data["password"] user.set_password(new_password) user.save() @@ -696,7 +694,7 @@ def change_password(request): 'hasher': hasher, }) - + @login_required @person_required def change_username(request): @@ -764,6 +762,28 @@ def clean(self): ) return super().clean() + def confirm_login_allowed(self, user): + """Check whether a successfully authenticated user is permitted to log in""" + super().confirm_login_allowed(user) + # Optionally enforce password validation + if getattr(settings, "PASSWORD_POLICY_ENFORCE_AT_LOGIN", False): + try: + password_validation.validate_password( + self.cleaned_data["password"], user + ) + except ValidationError: + raise ValidationError( + # dict mapping field to error / error list + { + "__all__": ValidationError( + 'You entered your password correctly, but it does not ' + 'meet our current length and complexity requirements. ' + 'Please use the "Forgot your password?" button below to ' + 'set a new password for your account.' + ), + } + ) + class AnyEmailLoginView(LoginView): """LoginView that allows any email address as the username @@ -779,7 +799,7 @@ def form_valid(self, form): logout(self.request) # should not be logged in yet, but just in case... return render(self.request, "registration/missing_person.html") return super().form_valid(form) - + @login_required @person_required diff --git a/ietf/ietfauth/widgets.py b/ietf/ietfauth/widgets.py index c9a0523402..fd7fa16726 100644 --- a/ietf/ietfauth/widgets.py +++ b/ietf/ietfauth/widgets.py @@ -39,18 +39,19 @@ def render(self, name, value, attrs=None, renderer=None): strength_markup = """
-
+
- """ % ( + """.format( _("Warning"), _( 'This password would take to crack.' diff --git a/ietf/ipr/admin.py b/ietf/ipr/admin.py index afc1952d72..d6a320203b 100644 --- a/ietf/ipr/admin.py +++ b/ietf/ipr/admin.py @@ -1,13 +1,23 @@ -# Copyright The IETF Trust 2010-2020, All Rights Reserved +# Copyright The IETF Trust 2010-2025, All Rights Reserved # -*- coding: utf-8 -*- from django import forms from django.contrib import admin from ietf.name.models import DocRelationshipName -from ietf.ipr.models import (IprDisclosureBase, IprDocRel, IprEvent, - RelatedIpr, HolderIprDisclosure, ThirdPartyIprDisclosure, GenericIprDisclosure, - NonDocSpecificIprDisclosure, LegacyMigrationIprEvent) +from ietf.ipr.models import ( + IprDisclosureBase, + IprDocRel, + IprEvent, + RelatedIpr, + HolderIprDisclosure, + RemovedIprDisclosure, + ThirdPartyIprDisclosure, + GenericIprDisclosure, + NonDocSpecificIprDisclosure, + LegacyMigrationIprEvent, +) +from ietf.utils.admin import SaferTabularInline # ------------------------------------------------------ # ModelAdmins @@ -20,13 +30,13 @@ class Meta: 'sections':forms.TextInput, } -class IprDocRelInline(admin.TabularInline): +class IprDocRelInline(SaferTabularInline): model = IprDocRel form = IprDocRelAdminForm raw_id_fields = ['document'] extra = 1 -class RelatedIprInline(admin.TabularInline): +class RelatedIprInline(SaferTabularInline): model = RelatedIpr raw_id_fields = ['target'] fk_name = 'source' @@ -110,3 +120,9 @@ class LegacyMigrationIprEventAdmin(admin.ModelAdmin): list_filter = ['time', 'type', 'response_due'] raw_id_fields = ['by', 'disclosure', 'message', 'in_reply_to'] admin.site.register(LegacyMigrationIprEvent, LegacyMigrationIprEventAdmin) + +class RemovedIprDisclosureAdmin(admin.ModelAdmin): + pass + + +admin.site.register(RemovedIprDisclosure, RemovedIprDisclosureAdmin) diff --git a/ietf/ipr/mail.py b/ietf/ipr/mail.py index 167b11956c..9bef751b95 100644 --- a/ietf/ipr/mail.py +++ b/ietf/ipr/mail.py @@ -66,9 +66,9 @@ def utc_from_string(s): if date is None: return None elif is_aware(date): - return date.astimezone(datetime.timezone.utc) + return date.astimezone(datetime.UTC) else: - return date.replace(tzinfo=datetime.timezone.utc) + return date.replace(tzinfo=datetime.UTC) # ---------------------------------------------------------------- # Email Functions diff --git a/ietf/ipr/management/tests.py b/ietf/ipr/management/tests.py index d84b0cfef4..d7acd65042 100644 --- a/ietf/ipr/management/tests.py +++ b/ietf/ipr/management/tests.py @@ -1,7 +1,7 @@ # Copyright The IETF Trust 2021, All Rights Reserved # -*- coding: utf-8 -*- """Tests of ipr management commands""" -import mock +from unittest import mock import sys from django.core.management import call_command diff --git a/ietf/ipr/migrations/0005_removediprdisclosure.py b/ietf/ipr/migrations/0005_removediprdisclosure.py new file mode 100644 index 0000000000..400a264579 --- /dev/null +++ b/ietf/ipr/migrations/0005_removediprdisclosure.py @@ -0,0 +1,28 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("ipr", "0004_holderiprdisclosure_is_blanket_disclosure"), + ] + + operations = [ + migrations.CreateModel( + name="RemovedIprDisclosure", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("removed_id", models.PositiveBigIntegerField(unique=True)), + ("reason", models.TextField()), + ], + ), + ] diff --git a/ietf/ipr/migrations/0006_already_removed_ipr.py b/ietf/ipr/migrations/0006_already_removed_ipr.py new file mode 100644 index 0000000000..0e2dbc63eb --- /dev/null +++ b/ietf/ipr/migrations/0006_already_removed_ipr.py @@ -0,0 +1,24 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +from django.db import migrations + + +def forward(apps, schema_editor): + RemovedIprDisclosure = apps.get_model("ipr", "RemovedIprDisclosure") + for id in (6544, 6068): + RemovedIprDisclosure.objects.create( + removed_id=id, + reason="This IPR disclosure was removed as objectively false.", + ) + + +def reverse(apps, schema_editor): + RemovedIprDisclosure = apps.get_model("ipr", "RemovedIprDisclosure") + RemovedIprDisclosure.objects.all().delete() + + +class Migration(migrations.Migration): + dependencies = [ + ("ipr", "0005_removediprdisclosure"), + ] + + operations = [migrations.RunPython(forward, reverse)] diff --git a/ietf/ipr/models.py b/ietf/ipr/models.py index 2d81eb4b42..ea148c2704 100644 --- a/ietf/ipr/models.py +++ b/ietf/ipr/models.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2007-2023, All Rights Reserved +# Copyright The IETF Trust 2007-2025, All Rights Reserved # -*- coding: utf-8 -*- @@ -270,3 +270,7 @@ class LegacyMigrationIprEvent(IprEvent): """A subclass of IprEvent specifically for capturing contents of legacy_url_0, the text of a disclosure submitted by email""" pass + +class RemovedIprDisclosure(models.Model): + removed_id = models.PositiveBigIntegerField(unique=True) + reason = models.TextField() diff --git a/ietf/ipr/resources.py b/ietf/ipr/resources.py index 0d8421cdec..c4d2c436e6 100644 --- a/ietf/ipr/resources.py +++ b/ietf/ipr/resources.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2015-2019, All Rights Reserved +# Copyright The IETF Trust 2015-2025, All Rights Reserved # -*- coding: utf-8 -*- # Autogenerated by the mkresources management command 2015-03-21 14:05 PDT @@ -11,7 +11,7 @@ from ietf import api -from ietf.ipr.models import ( IprDisclosureBase, IprDocRel, HolderIprDisclosure, ThirdPartyIprDisclosure, +from ietf.ipr.models import ( IprDisclosureBase, IprDocRel, HolderIprDisclosure, RemovedIprDisclosure, ThirdPartyIprDisclosure, RelatedIpr, NonDocSpecificIprDisclosure, GenericIprDisclosure, IprEvent, LegacyMigrationIprEvent ) from ietf.person.resources import PersonResource @@ -295,3 +295,18 @@ class Meta: } api.ipr.register(LegacyMigrationIprEventResource()) + + +class RemovedIprDisclosureResource(ModelResource): + class Meta: + queryset = RemovedIprDisclosure.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'removediprdisclosure' + ordering = ['id', ] + filtering = { + "id": ALL, + "removed_id": ALL, + "reason": ALL, + } +api.ipr.register(RemovedIprDisclosureResource()) diff --git a/ietf/ipr/tests.py b/ietf/ipr/tests.py index e0d00b5d1a..53a599e2de 100644 --- a/ietf/ipr/tests.py +++ b/ietf/ipr/tests.py @@ -3,7 +3,8 @@ import datetime -import mock +import json +from unittest import mock import re from pyquery import PyQuery @@ -15,6 +16,8 @@ from django.urls import reverse as urlreverse from django.utils import timezone +from django.db.models import Max + import debug # pyflakes:ignore from ietf.api.views import EmailIngestionError @@ -38,13 +41,14 @@ from ietf.ipr.forms import DraftForm, HolderIprDisclosureForm from ietf.ipr.mail import (process_response_email, get_reply_to, get_update_submitter_emails, get_pseudo_submitter, get_holders, get_update_cc_addrs, UndeliverableIprResponseError) -from ietf.ipr.models import (IprDisclosureBase, GenericIprDisclosure, HolderIprDisclosure, +from ietf.ipr.models import (IprDisclosureBase, GenericIprDisclosure, HolderIprDisclosure, RemovedIprDisclosure, ThirdPartyIprDisclosure, IprEvent) from ietf.ipr.templatetags.ipr_filters import no_revisions_message from ietf.ipr.utils import get_genitive, get_ipr_summary, ingest_response_email from ietf.mailtrigger.utils import gather_address_lists from ietf.message.factories import MessageFactory from ietf.message.models import Message +from ietf.person.factories import PersonFactory from ietf.utils.mail import outbox, empty_outbox, get_payload_text from ietf.utils.test_utils import TestCase, login_testing_unauthorized from ietf.utils.text import text_to_dict @@ -125,6 +129,26 @@ def test_showlist(self): self.assertContains(r, "removed as objectively false") ipr.delete() + def test_show_delete(self): + ipr = HolderIprDisclosureFactory() + removed = RemovedIprDisclosure.objects.create( + removed_id=ipr.pk, reason="Removed for reasons" + ) + url = urlreverse("ietf.ipr.views.show", kwargs=dict(id=removed.removed_id)) + r = self.client.get(url) + self.assertContains(r, "Removed for reasons") + q = PyQuery(r.content) + self.assertEqual(len(q("#deletion_warning")), 0) + self.client.login(username="secretary", password="secretary+password") + r = self.client.get(url) + self.assertContains(r, "Removed for reasons") + q = PyQuery(r.content) + self.assertEqual(len(q("#deletion_warning")), 1) + ipr.delete() + r = self.client.get(url) + self.assertContains(r, "Removed for reasons") + q = PyQuery(r.content) + self.assertEqual(len(q("#deletion_warning")), 0) def test_show_posted(self): ipr = HolderIprDisclosureFactory() @@ -1113,3 +1137,56 @@ def test_patent_details_required_unless_blanket(self): val = self.data.pop(pf) self.assertTrue(HolderIprDisclosureForm(data=self.data).is_valid()) self.data[pf] = val + +class JsonSnapshotTests(TestCase): + def test_json_snapshot(self): + h = HolderIprDisclosureFactory() + url = urlreverse("ietf.ipr.views.json_snapshot", kwargs=dict(id=h.id)) + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + dump = json.loads(r.content) + self.assertCountEqual( + [o["model"] for o in dump], + ["ipr.holderiprdisclosure", "ipr.iprdisclosurebase", "person.person"], + ) + h.docs.add(WgRfcFactory()) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + dump = json.loads(r.content) + self.assertCountEqual( + [o["model"] for o in dump], + [ + "ipr.holderiprdisclosure", + "ipr.iprdisclosurebase", + "ipr.iprdocrel", + "person.person", + ], + ) + IprEventFactory( + disclosure=h, + message=MessageFactory(by=PersonFactory()), + in_reply_to=MessageFactory(), + ) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + dump = json.loads(r.content) + self.assertCountEqual( + [o["model"] for o in dump], + [ + "ipr.holderiprdisclosure", + "ipr.iprdisclosurebase", + "ipr.iprdocrel", + "ipr.iprevent", + "message.message", + "message.message", + "person.person", + "person.person", + "person.person", + "person.person", + ], + ) + no_such_ipr_id = IprDisclosureBase.objects.aggregate(Max("id"))["id__max"] + 1 + url = urlreverse("ietf.ipr.views.json_snapshot", kwargs=dict(id=no_such_ipr_id)) + r = self.client.get(url) + self.assertEqual(r.status_code, 404) diff --git a/ietf/ipr/urls.py b/ietf/ipr/urls.py index 84ed04a66b..2c8a26c624 100644 --- a/ietf/ipr/urls.py +++ b/ietf/ipr/urls.py @@ -21,6 +21,7 @@ url(r'^(?P\d+)/notify/(?Pupdate|posted)/$', views.notify), url(r'^(?P\d+)/post/$', views.post), url(r'^(?P\d+)/state/$', views.state), + url(r'^(?P\d+)/json-snapshot/$', views.json_snapshot), url(r'^update/$', RedirectView.as_view(url=reverse_lazy('ietf.ipr.views.showlist'), permanent=True)), url(r'^update/(?P\d+)/$', views.update), url(r'^new-(?P<_type>(specific|generic|general|third-party))/$', views.new), diff --git a/ietf/ipr/utils.py b/ietf/ipr/utils.py index 7e569a1d1d..bcbb052260 100644 --- a/ietf/ipr/utils.py +++ b/ietf/ipr/utils.py @@ -1,11 +1,16 @@ -# Copyright The IETF Trust 2014-2020, All Rights Reserved +# Copyright The IETF Trust 2014-2025, All Rights Reserved # -*- coding: utf-8 -*- +import json +import debug # pyflakes:ignore + from textwrap import dedent +from django.core import serializers + from ietf.ipr.mail import process_response_email, UndeliverableIprResponseError -import debug # pyflakes:ignore +from ietf.ipr.models import IprDocRel def get_genitive(name): """Return the genitive form of name""" @@ -85,3 +90,18 @@ def ingest_response_email(message: bytes): email_original_message=message, email_attach_traceback=True, ) from err + +def json_dump_disclosure(disclosure): + objs = set() + objs.add(disclosure) + objs.add(disclosure.iprdisclosurebase_ptr) + objs.add(disclosure.by) + objs.update(IprDocRel.objects.filter(disclosure=disclosure)) + objs.update(disclosure.iprevent_set.all()) + objs.update([i.by for i in disclosure.iprevent_set.all()]) + objs.update([i.message for i in disclosure.iprevent_set.all() if i.message ]) + objs.update([i.message.by for i in disclosure.iprevent_set.all() if i.message ]) + objs.update([i.in_reply_to for i in disclosure.iprevent_set.all() if i.in_reply_to ]) + objs.update([i.in_reply_to.by for i in disclosure.iprevent_set.all() if i.in_reply_to ]) + objs = sorted(list(objs),key=lambda o:o.__class__.__name__) + return json.dumps(json.loads(serializers.serialize("json",objs)),indent=4) diff --git a/ietf/ipr/views.py b/ietf/ipr/views.py index 24453df2d2..0a43ff2c27 100644 --- a/ietf/ipr/views.py +++ b/ietf/ipr/views.py @@ -28,11 +28,11 @@ AddCommentForm, AddEmailForm, NotifyForm, StateForm, NonDocSpecificIprDisclosureForm, GenericIprDisclosureForm) from ietf.ipr.models import (IprDisclosureStateName, IprDisclosureBase, - HolderIprDisclosure, GenericIprDisclosure, ThirdPartyIprDisclosure, + HolderIprDisclosure, GenericIprDisclosure, RemovedIprDisclosure, ThirdPartyIprDisclosure, NonDocSpecificIprDisclosure, IprDocRel, RelatedIpr,IprEvent) from ietf.ipr.utils import (get_genitive, get_ipr_summary, - iprs_from_docs, related_docs) + iprs_from_docs, json_dump_disclosure, related_docs) from ietf.mailtrigger.utils import gather_address_lists from ietf.message.models import Message from ietf.message.utils import infer_message @@ -81,7 +81,8 @@ def get_document_emails(ipr): addrs = gather_address_lists('ipr_posted_on_doc',doc=doc).as_strings(compact=False) - author_names = ', '.join(a.person.name for a in doc.documentauthor_set.select_related("person")) + # Get a list of author names for the salutation in the body of the email + author_names = ', '.join(doc.author_names()) context = dict( settings=settings, @@ -152,13 +153,13 @@ def ipr_rfc_number(disclosureDate, thirdPartyDisclosureFlag): # RFC publication date comes from the RFC Editor announcement ipr_rfc_pub_datetime = { - 1310 : datetime.datetime(1992, 3, 13, 0, 0, tzinfo=datetime.timezone.utc), - 1802 : datetime.datetime(1994, 3, 23, 0, 0, tzinfo=datetime.timezone.utc), - 2026 : datetime.datetime(1996, 10, 29, 0, 0, tzinfo=datetime.timezone.utc), - 3668 : datetime.datetime(2004, 2, 18, 0, 0, tzinfo=datetime.timezone.utc), - 3979 : datetime.datetime(2005, 3, 2, 2, 23, tzinfo=datetime.timezone.utc), - 4879 : datetime.datetime(2007, 4, 10, 18, 21, tzinfo=datetime.timezone.utc), - 8179 : datetime.datetime(2017, 5, 31, 23, 1, tzinfo=datetime.timezone.utc), + 1310 : datetime.datetime(1992, 3, 13, 0, 0, tzinfo=datetime.UTC), + 1802 : datetime.datetime(1994, 3, 23, 0, 0, tzinfo=datetime.UTC), + 2026 : datetime.datetime(1996, 10, 29, 0, 0, tzinfo=datetime.UTC), + 3668 : datetime.datetime(2004, 2, 18, 0, 0, tzinfo=datetime.UTC), + 3979 : datetime.datetime(2005, 3, 2, 2, 23, tzinfo=datetime.UTC), + 4879 : datetime.datetime(2007, 4, 10, 18, 21, tzinfo=datetime.UTC), + 8179 : datetime.datetime(2017, 5, 31, 23, 1, tzinfo=datetime.UTC), } if disclosureDate < ipr_rfc_pub_datetime[1310]: @@ -817,7 +818,14 @@ def get_details_tabs(ipr, selected): def show(request, id): """View of individual declaration""" - ipr = get_object_or_404(IprDisclosureBase, id=id).get_child() + ipr = IprDisclosureBase.objects.filter(id=id) + removed = RemovedIprDisclosure.objects.filter(removed_id=id) + if removed.exists(): + return render(request, "ipr/deleted.html", {"removed": removed.get(), "ipr": ipr}) + if not ipr.exists(): + raise Http404 + else: + ipr = ipr.get().get_child() if not has_role(request.user, 'Secretariat'): if ipr.state.slug in ['removed', 'removed_objfalse']: return render(request, "ipr/removed.html", { @@ -901,3 +909,8 @@ def update(request, id): child = ipr.get_child() type = class_to_type[child.__class__.__name__] return new(request, type, updates=id) + +@role_required("Secretariat") +def json_snapshot(request, id): + obj = get_object_or_404(IprDisclosureBase,id=id).get_child() + return HttpResponse(json_dump_disclosure(obj),content_type="application/json") diff --git a/ietf/liaisons/admin.py b/ietf/liaisons/admin.py index c7cb7a4dae..d873cce536 100644 --- a/ietf/liaisons/admin.py +++ b/ietf/liaisons/admin.py @@ -7,15 +7,16 @@ from ietf.liaisons.models import ( LiaisonStatement, LiaisonStatementEvent, RelatedLiaisonStatement, LiaisonStatementAttachment ) +from ietf.utils.admin import SaferTabularInline -class RelatedLiaisonStatementInline(admin.TabularInline): +class RelatedLiaisonStatementInline(SaferTabularInline): model = RelatedLiaisonStatement fk_name = 'source' raw_id_fields = ['target'] extra = 1 -class LiaisonStatementAttachmentInline(admin.TabularInline): +class LiaisonStatementAttachmentInline(SaferTabularInline): model = LiaisonStatementAttachment raw_id_fields = ['document'] extra = 1 @@ -24,7 +25,7 @@ class LiaisonStatementAdmin(admin.ModelAdmin): list_display = ['id', 'title', 'submitted', 'from_groups_short_display', 'purpose', 'related_to'] list_display_links = ['id', 'title'] ordering = ('title', ) - raw_id_fields = ('from_contact', 'attachments', 'from_groups', 'to_groups') + raw_id_fields = ('attachments', 'from_groups', 'to_groups') #filter_horizontal = ('from_groups', 'to_groups') inlines = [ RelatedLiaisonStatementInline, LiaisonStatementAttachmentInline ] @@ -50,4 +51,4 @@ class LiaisonStatementEventAdmin(admin.ModelAdmin): raw_id_fields = ["statement", "by"] admin.site.register(LiaisonStatement, LiaisonStatementAdmin) -admin.site.register(LiaisonStatementEvent, LiaisonStatementEventAdmin) \ No newline at end of file +admin.site.register(LiaisonStatementEvent, LiaisonStatementEventAdmin) diff --git a/ietf/liaisons/factories.py b/ietf/liaisons/factories.py index 6d93cf8cd2..ca588236e3 100644 --- a/ietf/liaisons/factories.py +++ b/ietf/liaisons/factories.py @@ -9,7 +9,7 @@ class Meta: skip_postgeneration_save = True title = factory.Faker('sentence') - from_contact = factory.SubFactory('ietf.person.factories.EmailFactory') + from_contact = factory.Faker('email') purpose_id = 'comment' body = factory.Faker('paragraph') state_id = 'posted' diff --git a/ietf/liaisons/forms.py b/ietf/liaisons/forms.py index 1af29044b3..6ceda5ad38 100644 --- a/ietf/liaisons/forms.py +++ b/ietf/liaisons/forms.py @@ -3,38 +3,33 @@ import io -import os import operator - -from typing import Union # pyflakes:ignore - +import os from email.utils import parseaddr +from functools import reduce +from typing import Union, Optional # pyflakes:ignore from django import forms from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist, ValidationError -from django.forms.utils import ErrorList -from django.db.models import Q -#from django.forms.widgets import RadioFieldRenderer +from django.core.exceptions import ValidationError from django.core.validators import validate_email +from django.db.models import Q, QuerySet +from django.forms.utils import ErrorList from django_stubs_ext import QuerySetAny -import debug # pyflakes:ignore - +from ietf.doc.models import Document +from ietf.group.models import Group from ietf.ietfauth.utils import has_role -from ietf.name.models import DocRelationshipName -from ietf.liaisons.utils import get_person_for_user,is_authorized_individual -from ietf.liaisons.widgets import ButtonWidget,ShowAttachmentsWidget -from ietf.liaisons.models import (LiaisonStatement, - LiaisonStatementEvent,LiaisonStatementAttachment,LiaisonStatementPurposeName) from ietf.liaisons.fields import SearchableLiaisonStatementsField -from ietf.group.models import Group -from ietf.person.models import Email -from ietf.person.fields import SearchableEmailField -from ietf.doc.models import Document +from ietf.liaisons.models import (LiaisonStatement, + LiaisonStatementEvent, LiaisonStatementAttachment, LiaisonStatementPurposeName) +from ietf.liaisons.utils import get_person_for_user, OUTGOING_LIAISON_ROLES, \ + INCOMING_LIAISON_ROLES +from ietf.liaisons.widgets import ButtonWidget, ShowAttachmentsWidget +from ietf.name.models import DocRelationshipName +from ietf.person.models import Person from ietf.utils.fields import DatepickerDateField, ModelMultipleChoiceField from ietf.utils.timezone import date_today, datetime_from_date, DEADLINE_TZINFO -from functools import reduce ''' NOTES: @@ -51,45 +46,105 @@ def liaison_manager_sdos(person): return Group.objects.filter(type="sdo", state="active", role__person=person, role__name="liaiman").distinct() + def flatten_choices(choices): - '''Returns a flat choice list given one with option groups defined''' + """Returns a flat choice list given one with option groups defined + + n.b., Django allows mixing grouped options and top-level options. This helper only supports + the non-mixed case where every option is in an option group. + """ flat = [] - for optgroup,options in choices: + for optgroup, options in choices: flat.extend(options) return flat + + +def choices_from_group_queryset(groups: QuerySet[Group]): + """Get choices list for internal IETF groups user is authorized to select -def get_internal_choices(user): - '''Returns the set of internal IETF groups the user has permissions for, as a list - of choices suitable for use in a select widget. If user == None, all active internal - groups are included.''' + Returns a grouped list of choices suitable for use with a ChoiceField. If user is None, + includes all groups. + """ + main = [] + areas = [] + wgs = [] + for g in groups.distinct().order_by("acronym"): + if g.acronym in ("ietf", "iesg", "iab"): + main.append((g.pk, f"The {g.acronym.upper()}")) + elif g.type_id == "area": + areas.append((g.pk, f"{g.acronym} - {g.name}")) + elif g.type_id == "wg": + wgs.append((g.pk, f"{g.acronym} - {g.name}")) choices = [] - groups = get_groups_for_person(user.person if user else None) - main = [ (g.pk, 'The {}'.format(g.acronym.upper())) for g in groups.filter(acronym__in=('ietf','iesg','iab')) ] - areas = [ (g.pk, '{} - {}'.format(g.acronym,g.name)) for g in groups.filter(type='area') ] - wgs = [ (g.pk, '{} - {}'.format(g.acronym,g.name)) for g in groups.filter(type='wg') ] - choices.append(('Main IETF Entities', main)) - choices.append(('IETF Areas', areas)) - choices.append(('IETF Working Groups', wgs )) + if len(main) > 0: + choices.append(("Main IETF Entities", main)) + if len(areas) > 0: + choices.append(("IETF Areas", areas)) + if len(wgs) > 0: + choices.append(("IETF Working Groups", wgs)) return choices -def get_groups_for_person(person): - '''Returns queryset of internal Groups the person has interesting roles in. - This is a refactor of IETFHierarchyManager.get_entities_for_person(). If Person - is None or Secretariat or Liaison Manager all internal IETF groups are returned. - ''' - if person == None or has_role(person.user, "Secretariat") or has_role(person.user, "Liaison Manager"): - # collect all internal IETF groups - queries = [Q(acronym__in=('ietf','iesg','iab')), - Q(type='area',state='active'), - Q(type='wg',state='active')] + +def all_internal_groups(): + """Get a queryset of all IETF groups suitable for LS To/From assignment""" + return Group.objects.filter( + Q(acronym__in=("ietf", "iesg", "iab")) + | Q(type="area", state="active") + | Q(type="wg", state="active") + ).distinct() + + +def internal_groups_for_person(person: Optional[Person]): + """Get a queryset of IETF groups suitable for LS To/From assignment by person""" + if person is None: + return Group.objects.none() # no person = no roles + + if has_role( + person.user, + ( + "Secretariat", + "IETF Chair", + "IAB Chair", + "Liaison Manager", + "Liaison Coordinator", + "Authorized Individual", + ), + ): + return all_internal_groups() + # Interesting roles, as Group queries + queries = [ + Q(role__person=person, role__name="chair", acronym="ietf"), + Q(role__person=person, role__name="chair", acronym="iab"), + Q(role__person=person, role__name="ad", type="area", state="active"), + Q( + role__person=person, + role__name__in=("chair", "secretary"), + type="wg", + state="active", + ), + Q( + parent__role__person=person, + parent__role__name="ad", + type="wg", + state="active", + ), + ] + if has_role(person.user, "Area Director"): + queries.append(Q(acronym__in=("ietf", "iesg"))) # AD can also choose these + return Group.objects.filter(reduce(operator.or_, queries)).distinct() + + +def external_groups_for_person(person): + """Get a queryset of external groups suitable for LS To/From assignment by person""" + filter_expr = Q(pk__in=[]) # start with no groups + # These roles can add all external sdo groups + if has_role(person.user, set(INCOMING_LIAISON_ROLES + OUTGOING_LIAISON_ROLES) - {"Liaison Manager", "Authorized Individual"}): + filter_expr |= Q(type="sdo") else: - # Interesting roles, as Group queries - queries = [Q(role__person=person,role__name='chair',acronym='ietf'), - Q(role__person=person,role__name__in=('chair','execdir'),acronym='iab'), - Q(role__person=person,role__name='ad',type='area',state='active'), - Q(role__person=person,role__name__in=('chair','secretary'),type='wg',state='active'), - Q(parent__role__person=person,parent__role__name='ad',type='wg',state='active')] - return Group.objects.filter(reduce(operator.or_,queries)).order_by('acronym').distinct() + # The person cannot add all external sdo groups; add any for which they are Liaison Manager + filter_expr |= Q(type="sdo", role__person=person, role__name__in=["auth", "liaiman"]) + return Group.objects.filter(state="active").filter(filter_expr).distinct().order_by("name") + def liaison_form_factory(request, type=None, **kwargs): """Returns appropriate Liaison entry form""" @@ -154,7 +209,7 @@ def get_results(self): query = self.cleaned_data.get('text') if query: q = (Q(title__icontains=query) | - Q(from_contact__address__icontains=query) | + Q(from_contact__icontains=query) | Q(to_contacts__icontains=query) | Q(other_identifiers__icontains=query) | Q(body__icontains=query) | @@ -216,13 +271,8 @@ class LiaisonModelForm(forms.ModelForm): '''Specify fields which require a custom widget or that are not part of the model. ''' from_groups = ModelMultipleChoiceField(queryset=Group.objects.all(),label='Groups',required=False) - from_groups.widget.attrs["class"] = "select2-field" - from_groups.widget.attrs['data-minimum-input-length'] = 0 - from_contact = forms.EmailField() # type: Union[forms.EmailField, SearchableEmailField] to_contacts = forms.CharField(label="Contacts", widget=forms.Textarea(attrs={'rows':'3', }), strip=False) to_groups = ModelMultipleChoiceField(queryset=Group.objects,label='Groups',required=False) - to_groups.widget.attrs["class"] = "select2-field" - to_groups.widget.attrs['data-minimum-input-length'] = 0 deadline = DatepickerDateField(date_format="yyyy-mm-dd", picker_settings={"autoclose": "1" }, label='Deadline', required=True) related_to = SearchableLiaisonStatementsField(label='Related Liaison Statement', required=False) submitted_date = DatepickerDateField(date_format="yyyy-mm-dd", picker_settings={"autoclose": "1" }, label='Submission date', required=True, initial=lambda: date_today(DEADLINE_TZINFO)) @@ -245,13 +295,17 @@ def __init__(self, user, *args, **kwargs): self.person = get_person_for_user(user) self.is_new = not self.instance.pk + self.fields["from_groups"].widget.attrs["class"] = "select2-field" + self.fields["from_groups"].widget.attrs["data-minimum-input-length"] = 0 self.fields["from_groups"].widget.attrs["data-placeholder"] = "Type in name to search for group" + self.fields["to_groups"].widget.attrs["class"] = "select2-field" + self.fields["to_groups"].widget.attrs["data-minimum-input-length"] = 0 self.fields["to_groups"].widget.attrs["data-placeholder"] = "Type in name to search for group" self.fields["to_contacts"].label = 'Contacts' self.fields["other_identifiers"].widget.attrs["rows"] = 2 - + # add email validators - for field in ['from_contact','to_contacts','technical_contacts','action_holder_contacts','cc_contacts']: + for field in ['to_contacts','technical_contacts','action_holder_contacts','cc_contacts']: if field in self.fields: self.fields[field].validators.append(validate_emails) @@ -270,18 +324,6 @@ def clean_to_groups(self): raise forms.ValidationError('You must specify a To Group') return to_groups - def clean_from_contact(self): - contact = self.cleaned_data.get('from_contact') - from_groups = self.cleaned_data.get('from_groups') - try: - email = Email.objects.get(address=contact) - if not email.origin: - email.origin = "liaison: %s" % (','.join([ g.acronym for g in from_groups.all() ])) - email.save() - except ObjectDoesNotExist: - raise forms.ValidationError('Email address does not exist') - return email - # Note to future person: This is the wrong place to fix the new lines # in cc_contacts and to_contacts. Those belong in the save function. # Or at least somewhere other than here. @@ -424,42 +466,39 @@ def set_to_fields(self): assert NotImplemented class IncomingLiaisonForm(LiaisonModelForm): - def clean(self): - if 'send' in list(self.data.keys()) and self.get_post_only(): - raise forms.ValidationError('As an IETF Liaison Manager you can not send incoming liaison statements, you only can post them') - return super(IncomingLiaisonForm, self).clean() def is_approved(self): '''Incoming Liaison Statements do not required approval''' return True - def get_post_only(self): - from_groups = self.cleaned_data.get('from_groups') - if has_role(self.user, "Secretariat") or is_authorized_individual(self.user,from_groups): - return False - return True - def set_from_fields(self): - '''Set from_groups and from_contact options and initial value based on user - accessing the form.''' - if has_role(self.user, "Secretariat"): - queryset = Group.objects.filter(type="sdo", state="active").order_by('name') - else: - queryset = Group.objects.filter(type="sdo", state="active", role__person=self.person, role__name__in=("liaiman", "auth")).distinct().order_by('name') - self.fields['from_contact'].initial = self.person.role_set.filter(group=queryset[0]).first().email.address - self.fields['from_contact'].widget.attrs['disabled'] = True - self.fields['from_groups'].queryset = queryset - self.fields['from_groups'].widget.submitter = str(self.person) - + """Configure from "From" fields based on user roles""" + qs = external_groups_for_person(self.person) + self.fields["from_groups"].queryset = qs + self.fields["from_groups"].widget.submitter = str(self.person) # if there's only one possibility make it the default - if len(queryset) == 1: - self.fields['from_groups'].initial = queryset + if len(qs) == 1: + self.fields['from_groups'].initial = qs + + # Note that the IAB chair currently doesn't get to work with incoming liaison statements + + # Removing this block at the request of the IAB - as a workaround until the new liaison tool is + # create, anyone with access to the form can set any from_contact value + # + # if not ( + # has_role(self.user, "Secretariat") + # or has_role(self.user, "Liaison Coordinator") + # ): + # self.fields["from_contact"].initial = ( + # self.person.role_set.filter(group=qs[0]).first().email.formatted_email() + # ) + # self.fields["from_contact"].widget.attrs["disabled"] = True def set_to_fields(self): '''Set to_groups and to_contacts options and initial value based on user accessing the form. For incoming Liaisons, to_groups choices is the full set. ''' - self.fields['to_groups'].choices = get_internal_choices(None) + self.fields['to_groups'].choices = choices_from_group_queryset(all_internal_groups()) class OutgoingLiaisonForm(LiaisonModelForm): @@ -473,46 +512,56 @@ def is_approved(self): return self.cleaned_data['approved'] def set_from_fields(self): - '''Set from_groups and from_contact options and initial value based on user - accessing the form''' - choices = get_internal_choices(self.user) - self.fields['from_groups'].choices = choices - - # set initial value if only one entry - flat_choices = flatten_choices(choices) + """Configure from "From" fields based on user roles""" + self.set_from_groups_field() + self.set_from_contact_field() + + def set_from_groups_field(self): + """Configure the from_groups field based on roles""" + grouped_choices = choices_from_group_queryset(internal_groups_for_person(self.person)) + flat_choices = flatten_choices(grouped_choices) if len(flat_choices) == 1: - self.fields['from_groups'].initial = [flat_choices[0][0]] - - if has_role(self.user, "Secretariat"): - self.fields['from_contact'] = SearchableEmailField(only_users=True) # secretariat can edit this field! - return - - if self.person.role_set.filter(name='liaiman',group__state='active'): - email = self.person.role_set.filter(name='liaiman',group__state='active').first().email.address - elif self.person.role_set.filter(name__in=('ad','chair'),group__state='active'): - email = self.person.role_set.filter(name__in=('ad','chair'),group__state='active').first().email.address + self.fields["from_groups"].choices = flat_choices + self.fields["from_groups"].initial = [flat_choices[0][0]] else: - email = self.person.email_address() + self.fields["from_groups"].choices = grouped_choices - # Non-secretariat user cannot change the from_contact field. Fill in its value. + def set_from_contact_field(self): + """Configure the from_contact field based on user roles""" + # Secretariat can set this to any valid address but gets no default + if has_role(self.user, "Secretariat"): + return + elif has_role(self.user, ["IAB Chair", "Liaison Coordinator"]): + self.fields["from_contact"].initial = "IAB Chair " + return + elif has_role(self.user, "IETF Chair"): + self.fields["from_contact"].initial = "IETF Chair " + return + # ... others have it set to the correct value and cannot change it self.fields['from_contact'].disabled = True - self.fields['from_contact'].initial = email - - def set_to_fields(self): - '''Set to_groups and to_contacts options and initial value based on user - accessing the form''' - # set options. if the user is a Liaison Manager and nothing more, reduce set to his SDOs - if has_role(self.user, "Liaison Manager") and not self.person.role_set.filter(name__in=('ad','chair'),group__state='active'): - queryset = Group.objects.filter(type="sdo", state="active", role__person=self.person, role__name="liaiman").distinct().order_by('name') + # Set up the querysets we might use - only evaluated as needed + liaison_manager_role = self.person.role_set.filter(name="liaiman", group__state="active") + chair_or_ad_role = self.person.role_set.filter( + name__in=("ad", "chair"), group__state="active" + ) + if liaison_manager_role.exists(): + from_contact_email = liaison_manager_role.first().email + elif chair_or_ad_role.exists(): + from_contact_email = chair_or_ad_role.first().email else: - # get all outgoing entities - queryset = Group.objects.filter(type="sdo", state="active").order_by('name') + from_contact_email = self.person.email() + self.fields['from_contact'].initial = from_contact_email.formatted_email() - self.fields['to_groups'].queryset = queryset + def set_to_fields(self): + """Configure the "To" fields based on user roles""" + qs = external_groups_for_person(self.person) + self.fields['to_groups'].queryset = qs # set initial if has_role(self.user, "Liaison Manager"): - self.fields['to_groups'].initial = [queryset.first()] + self.fields['to_groups'].initial = [ + qs.filter(role__person=self.person, role__name="liaiman").first() + ] class EditLiaisonForm(LiaisonModelForm): @@ -533,32 +582,20 @@ def save(self, *args, **kwargs): return self.instance def set_from_fields(self): - '''Set from_groups and from_contact options and initial value based on user - accessing the form.''' + """Configure from "From" fields based on user roles""" if self.instance.is_outgoing(): - self.fields['from_groups'].choices = get_internal_choices(self.user) + self.fields['from_groups'].choices = choices_from_group_queryset(internal_groups_for_person(self.person)) else: - if has_role(self.user, "Secretariat"): - queryset = Group.objects.filter(type="sdo").order_by('name') - else: - queryset = Group.objects.filter(type="sdo", role__person=self.person, role__name__in=("liaiman", "auth")).distinct().order_by('name') + self.fields["from_groups"].queryset = external_groups_for_person(self.person) + if not has_role(self.user, "Secretariat"): self.fields['from_contact'].widget.attrs['disabled'] = True - self.fields['from_groups'].queryset = queryset def set_to_fields(self): - '''Set to_groups and to_contacts options and initial value based on user - accessing the form. For incoming Liaisons, to_groups choices is the full set. - ''' + """Configure the "To" fields based on user roles""" if self.instance.is_outgoing(): - # if the user is a Liaison Manager and nothing more, reduce to set to his SDOs - if has_role(self.user, "Liaison Manager") and not self.person.role_set.filter(name__in=('ad','chair'),group__state='active'): - queryset = Group.objects.filter(type="sdo", role__person=self.person, role__name="liaiman").distinct().order_by('name') - else: - # get all outgoing entities - queryset = Group.objects.filter(type="sdo").order_by('name') - self.fields['to_groups'].queryset = queryset + self.fields['to_groups'].queryset = external_groups_for_person(self.person) else: - self.fields['to_groups'].choices = get_internal_choices(None) + self.fields['to_groups'].choices = choices_from_group_queryset(all_internal_groups()) class EditAttachmentForm(forms.Form): diff --git a/ietf/liaisons/mails.py b/ietf/liaisons/mails.py index 8708c8a078..878aada576 100644 --- a/ietf/liaisons/mails.py +++ b/ietf/liaisons/mails.py @@ -14,7 +14,10 @@ def send_liaison_by_email(request, liaison): subject = 'New Liaison Statement, "%s"' % (liaison.title) from_email = settings.LIAISON_UNIVERSAL_FROM - (to_email, cc) = gather_address_lists('liaison_statement_posted',liaison=liaison) + if liaison.is_outgoing(): + (to_email, cc) = gather_address_lists('liaison_statement_posted_outgoing',liaison=liaison) + else: + (to_email, cc) = gather_address_lists('liaison_statement_posted_incoming',liaison=liaison) bcc = ['statements@ietf.org'] body = render_to_string('liaisons/liaison_mail.txt', dict(liaison=liaison)) diff --git a/ietf/liaisons/migrations/0002_alter_liaisonstatement_response_contacts.py b/ietf/liaisons/migrations/0002_alter_liaisonstatement_response_contacts.py new file mode 100644 index 0000000000..ac0a11101b --- /dev/null +++ b/ietf/liaisons/migrations/0002_alter_liaisonstatement_response_contacts.py @@ -0,0 +1,20 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("liaisons", "0001_initial"), + ] + + operations = [ + migrations.AlterField( + model_name="liaisonstatement", + name="response_contacts", + field=models.TextField( + blank=True, help_text="Where to send a response", max_length=1024 + ), + ), + ] diff --git a/ietf/liaisons/migrations/0003_liaisonstatement_from_contact_tmp.py b/ietf/liaisons/migrations/0003_liaisonstatement_from_contact_tmp.py new file mode 100644 index 0000000000..de2ce7ff59 --- /dev/null +++ b/ietf/liaisons/migrations/0003_liaisonstatement_from_contact_tmp.py @@ -0,0 +1,22 @@ +# Copyright The IETF Trust 2025 All Rights Reserved +from django.db import migrations, models +import ietf.utils.validators + + +class Migration(migrations.Migration): + dependencies = [ + ("liaisons", "0002_alter_liaisonstatement_response_contacts"), + ] + + operations = [ + migrations.AddField( + model_name="liaisonstatement", + name="from_contact_tmp", + field=models.CharField( + blank=True, + help_text="Address of the formal sender of the statement", + max_length=512, + validators=[ietf.utils.validators.validate_mailbox_address], + ), + ), + ] diff --git a/ietf/liaisons/migrations/0004_populate_liaisonstatement_from_contact_tmp.py b/ietf/liaisons/migrations/0004_populate_liaisonstatement_from_contact_tmp.py new file mode 100644 index 0000000000..dbab326b0c --- /dev/null +++ b/ietf/liaisons/migrations/0004_populate_liaisonstatement_from_contact_tmp.py @@ -0,0 +1,60 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +from itertools import islice + +from django.db import migrations + +from ietf.person.name import plain_name +from ietf.utils.mail import formataddr +from ietf.utils.validators import validate_mailbox_address + + +def forward(apps, schema_editor): + def _formatted_email(email): + """Format an email address to match Email.formatted_email()""" + person = email.person + if person: + return formataddr( + ( + # inlined Person.plain_name(), minus the caching + person.plain if person.plain else plain_name(person.name), + email.address, + ) + ) + return email.address + + def _batched(iterable, n): + """Split an iterable into lists of length <= n + + (based on itertools example code for batched(), which is added in py312) + """ + iterator = iter(iterable) + batch = list(islice(iterator, n)) # consumes first n iterations + while batch: + yield batch + batch = list(islice(iterator, n)) # consumes next n iterations + + LiaisonStatement = apps.get_model("liaisons", "LiaisonStatement") + LiaisonStatement.objects.update(from_contact_tmp="") # ensure they're all blank + for batch in _batched( + LiaisonStatement.objects.exclude(from_contact=None).select_related( + "from_contact" + ), + 100, + ): + for ls in batch: + ls.from_contact_tmp = _formatted_email(ls.from_contact) + validate_mailbox_address( + ls.from_contact_tmp + ) # be sure it's permitted before we accept it + + LiaisonStatement.objects.bulk_update(batch, fields=["from_contact_tmp"]) + + +class Migration(migrations.Migration): + dependencies = [ + ("liaisons", "0003_liaisonstatement_from_contact_tmp"), + ] + + operations = [ + migrations.RunPython(forward), + ] diff --git a/ietf/liaisons/migrations/0005_replace_liaisonstatement_from_contact.py b/ietf/liaisons/migrations/0005_replace_liaisonstatement_from_contact.py new file mode 100644 index 0000000000..e1702ae3bc --- /dev/null +++ b/ietf/liaisons/migrations/0005_replace_liaisonstatement_from_contact.py @@ -0,0 +1,20 @@ +# Copyright The IETF Trust 2025 All Rights Reserved +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("liaisons", "0004_populate_liaisonstatement_from_contact_tmp"), + ] + + operations = [ + migrations.RemoveField( + model_name="liaisonstatement", + name="from_contact", + ), + migrations.RenameField( + model_name="liaisonstatement", + old_name="from_contact_tmp", + new_name="from_contact", + ), + ] diff --git a/ietf/liaisons/models.py b/ietf/liaisons/models.py index f357d6cf8c..a2d79ea476 100644 --- a/ietf/liaisons/models.py +++ b/ietf/liaisons/models.py @@ -7,13 +7,14 @@ from django.db import models from django.utils.text import slugify -from ietf.person.models import Email, Person +from ietf.person.models import Person from ietf.name.models import (LiaisonStatementPurposeName, LiaisonStatementState, LiaisonStatementEventTypeName, LiaisonStatementTagName, DocRelationshipName) from ietf.doc.models import Document from ietf.group.models import Group from ietf.utils.models import ForeignKey +from ietf.utils.validators import validate_mailbox_address # maps (previous state id, new state id) to event type id STATE_EVENT_MAPPING = { @@ -29,11 +30,16 @@ class LiaisonStatement(models.Model): title = models.CharField(max_length=255) from_groups = models.ManyToManyField(Group, blank=True, related_name='liaisonstatement_from_set') - from_contact = ForeignKey(Email, blank=True, null=True) + from_contact = models.CharField( + blank=True, + max_length=512, + help_text="Address of the formal sender of the statement", + validators=(validate_mailbox_address,) + ) to_groups = models.ManyToManyField(Group, blank=True, related_name='liaisonstatement_to_set') to_contacts = models.CharField(max_length=2000, help_text="Contacts at recipient group") - response_contacts = models.CharField(blank=True, max_length=255, help_text="Where to send a response") # RFC4053 + response_contacts = models.TextField(blank=True, max_length=1024, help_text="Where to send a response") # RFC4053 technical_contacts = models.CharField(blank=True, max_length=255, help_text="Who to contact for clarification") # RFC4053 action_holder_contacts = models.CharField(blank=True, max_length=255, help_text="Who makes sure action is completed") # incoming only? cc_contacts = models.TextField(blank=True) @@ -85,7 +91,7 @@ def name(self): if self.from_groups.count(): frm = ', '.join([i.acronym or i.name for i in self.from_groups.all()]) else: - frm = self.from_contact.person.name + frm = self.from_contact if self.to_groups.count(): to = ', '.join([i.acronym or i.name for i in self.to_groups.all()]) else: diff --git a/ietf/liaisons/resources.py b/ietf/liaisons/resources.py index 8f31ea3a64..02cd159a11 100644 --- a/ietf/liaisons/resources.py +++ b/ietf/liaisons/resources.py @@ -15,12 +15,10 @@ RelatedLiaisonStatement) -from ietf.person.resources import EmailResource from ietf.group.resources import GroupResource from ietf.name.resources import LiaisonStatementPurposeNameResource, LiaisonStatementTagNameResource, LiaisonStatementStateResource from ietf.doc.resources import DocumentResource class LiaisonStatementResource(ModelResource): - from_contact = ToOneField(EmailResource, 'from_contact', null=True) purpose = ToOneField(LiaisonStatementPurposeNameResource, 'purpose') state = ToOneField(LiaisonStatementStateResource, 'state') from_groups = ToManyField(GroupResource, 'from_groups', null=True) @@ -36,6 +34,7 @@ class Meta: filtering = { "id": ALL, "title": ALL, + "from_contact": ALL, "to_contacts": ALL, "response_contacts": ALL, "technical_contacts": ALL, @@ -44,9 +43,6 @@ class Meta: "deadline": ALL, "other_identifiers": ALL, "body": ALL, - "from_name": ALL, - "to_name": ALL, - "from_contact": ALL_WITH_RELATIONS, "purpose": ALL_WITH_RELATIONS, "state": ALL_WITH_RELATIONS, "from_groups": ALL_WITH_RELATIONS, diff --git a/ietf/liaisons/tests.py b/ietf/liaisons/tests.py index 1742687f14..e29045443f 100644 --- a/ietf/liaisons/tests.py +++ b/ietf/liaisons/tests.py @@ -110,65 +110,74 @@ def test_help_pages(self): self.assertEqual(self.client.get('/liaison/help/from_ietf/').status_code, 200) self.assertEqual(self.client.get('/liaison/help/to_ietf/').status_code, 200) + def test_list_other_sdo(self): + GroupFactory(type_id="sdo", state_id="conclude", acronym="third") + GroupFactory(type_id="sdo", state_id="active", acronym="second") + GroupFactory(type_id="sdo", state_id="active", acronym="first") + url = urlreverse("ietf.liaisons.views.list_other_sdo") + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + q = PyQuery(r.content) + self.assertEqual(len(q("h1")), 2) + first_td_elements_text = [e.text for e in q("tr").find("td:first-child a")] + self.assertEqual(first_td_elements_text, ["first", "second", "third"]) class UnitTests(TestCase): - def test_get_cc(self): - from ietf.liaisons.views import get_cc,EMAIL_ALIASES + def test_get_contacts_for_liaison_messages_for_group_primary(self): + from ietf.mailtrigger.utils import get_contacts_for_liaison_messages_for_group_primary,EMAIL_ALIASES # test IETF - cc = get_cc(Group.objects.get(acronym='ietf')) + cc = get_contacts_for_liaison_messages_for_group_primary(Group.objects.get(acronym='ietf')) self.assertTrue(EMAIL_ALIASES['IESG'] in cc) self.assertTrue(EMAIL_ALIASES['IETFCHAIR'] in cc) # test IAB - cc = get_cc(Group.objects.get(acronym='iab')) + cc = get_contacts_for_liaison_messages_for_group_primary(Group.objects.get(acronym='iab')) self.assertTrue(EMAIL_ALIASES['IAB'] in cc) self.assertTrue(EMAIL_ALIASES['IABCHAIR'] in cc) - self.assertTrue(EMAIL_ALIASES['IABEXECUTIVEDIRECTOR'] in cc) # test an Area area = Group.objects.filter(type='area').first() - cc = get_cc(area) + cc = get_contacts_for_liaison_messages_for_group_primary(area) self.assertTrue(EMAIL_ALIASES['IETFCHAIR'] in cc) self.assertTrue(contacts_from_roles([area.ad_role()]) in cc) # test a Working Group wg = Group.objects.filter(type='wg').first() - cc = get_cc(wg) + cc = get_contacts_for_liaison_messages_for_group_primary(wg) self.assertTrue(contacts_from_roles([wg.parent.ad_role()]) in cc) self.assertTrue(contacts_from_roles([wg.get_chair()]) in cc) # test an SDO sdo = RoleFactory(name_id='liaiman',group__type_id='sdo',).group - cc = get_cc(sdo) + cc = get_contacts_for_liaison_messages_for_group_primary(sdo) self.assertTrue(contacts_from_roles([sdo.role_set.filter(name='liaiman').first()]) in cc) # test a cc_contact role cc_contact_role = RoleFactory(name_id='liaison_cc_contact', group=sdo) - cc = get_cc(sdo) + cc = get_contacts_for_liaison_messages_for_group_primary(sdo) self.assertIn(contact_email_from_role(cc_contact_role), cc) - def test_get_contacts_for_group(self): - from ietf.liaisons.views import get_contacts_for_group, EMAIL_ALIASES + def test_get_contacts_for_liaison_messages_for_group_secondary(self): + from ietf.mailtrigger.utils import get_contacts_for_liaison_messages_for_group_secondary,EMAIL_ALIASES - # test explicit + # test explicit group contacts sdo = GroupFactory(type_id='sdo') contact_email = RoleFactory(name_id='liaison_contact', group=sdo).email.address - contacts = get_contacts_for_group(sdo) + contacts = get_contacts_for_liaison_messages_for_group_secondary(sdo) self.assertIsNotNone(contact_email) self.assertIn(contact_email, contacts) # test area area = Group.objects.filter(type='area').first() - contacts = get_contacts_for_group(area) + contacts = get_contacts_for_liaison_messages_for_group_secondary(area) self.assertTrue(area.ad_role().email.address in contacts) # test wg wg = Group.objects.filter(type='wg').first() - contacts = get_contacts_for_group(wg) + contacts = get_contacts_for_liaison_messages_for_group_secondary(wg) self.assertTrue(wg.get_chair().email.address in contacts) # test ietf - contacts = get_contacts_for_group(Group.objects.get(acronym='ietf')) + contacts = get_contacts_for_liaison_messages_for_group_secondary(Group.objects.get(acronym='ietf')) self.assertTrue(EMAIL_ALIASES['IETFCHAIR'] in contacts) # test iab - contacts = get_contacts_for_group(Group.objects.get(acronym='iab')) + contacts = get_contacts_for_liaison_messages_for_group_secondary(Group.objects.get(acronym='iab')) self.assertTrue(EMAIL_ALIASES['IABCHAIR'] in contacts) - self.assertTrue(EMAIL_ALIASES['IABEXECUTIVEDIRECTOR'] in contacts) # test iesg - contacts = get_contacts_for_group(Group.objects.get(acronym='iesg')) + contacts = get_contacts_for_liaison_messages_for_group_secondary(Group.objects.get(acronym='iesg')) self.assertTrue(EMAIL_ALIASES['IESG'] in contacts) def test_needs_approval(self): @@ -205,7 +214,6 @@ def test_ajax(self): self.assertEqual(r.status_code, 200) data = r.json() self.assertEqual(data["error"], False) - self.assertEqual(data["post_only"], False) self.assertTrue('cc' in data) self.assertTrue('needs_approval' in data) self.assertTrue('to_contacts' in data) @@ -365,6 +373,9 @@ def test_approval_process(self): self.assertEqual(len(q('form button[name=approved]')), 0) # check the detail page / authorized + r = self.client.post(url, dict(dead="1")) + self.assertEqual(r.status_code, 403) + mailbox_before = len(outbox) self.client.login(username="ulm-liaiman", password="ulm-liaiman+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) @@ -462,11 +473,12 @@ def test_edit_liaison(self): def test_incoming_access(self): - '''Ensure only Secretariat, Liaison Managers, and Authorized Individuals + '''Ensure only Secretariat, Liaison Managers, Liaison Coordinators, and Authorized Individuals have access to incoming liaisons. ''' sdo = RoleFactory(name_id='liaiman',group__type_id='sdo', person__user__username='ulm-liaiman').group RoleFactory(name_id='auth',group=sdo,person__user__username='ulm-auth') + RoleFactory(name_id='liaison_coordinator', group__acronym='iab', person__user__username='liaison-coordinator') stmt = LiaisonStatementFactory(from_groups=[sdo,]) LiaisonStatementEventFactory(statement=stmt,type_id='posted') RoleFactory(name_id='chair',person__user__username='marschairman',group__acronym='mars') @@ -499,6 +511,15 @@ def test_incoming_access(self): r = self.client.get(addurl) self.assertEqual(r.status_code, 200) + # Liaison Coordinator has access + self.client.login(username="liaison-coordinator", password="liaison-coordinator+password") + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + self.assertEqual(len(q('a.btn:contains("New incoming liaison")')), 1) + r = self.client.get(addurl) + self.assertEqual(r.status_code, 200) + # Authorized Individual has access self.client.login(username="ulm-auth", password="ulm-auth+password") r = self.client.get(url) @@ -521,9 +542,9 @@ def test_outgoing_access(self): sdo = RoleFactory(name_id='liaiman',group__type_id='sdo', person__user__username='ulm-liaiman').group RoleFactory(name_id='auth',group=sdo,person__user__username='ulm-auth') + RoleFactory(name_id='liaison_coordinator', group__acronym='iab', person__user__username='liaison-coordinator') mars = RoleFactory(name_id='chair',person__user__username='marschairman',group__acronym='mars').group RoleFactory(name_id='secr',group=mars,person__user__username='mars-secr') - RoleFactory(name_id='execdir',group=Group.objects.get(acronym='iab'),person__user__username='iab-execdir') url = urlreverse('ietf.liaisons.views.liaison_list') addurl = urlreverse('ietf.liaisons.views.liaison_add', kwargs={'type':'outgoing'}) @@ -581,17 +602,17 @@ def test_outgoing_access(self): r = self.client.get(addurl) self.assertEqual(r.status_code, 200) - # IAB Executive Director - self.assertTrue(self.client.login(username="iab-execdir", password="iab-execdir+password")) + # Liaison Manager has access + self.assertTrue(self.client.login(username="ulm-liaiman", password="ulm-liaiman+password")) r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertEqual(len(q("a.btn:contains('New outgoing liaison')")), 1) + self.assertEqual(len(q('a.btn:contains("New outgoing liaison")')), 1) r = self.client.get(addurl) self.assertEqual(r.status_code, 200) - # Liaison Manager has access - self.assertTrue(self.client.login(username="ulm-liaiman", password="ulm-liaiman+password")) + # Liaison Coordinator has access + self.assertTrue(self.client.login(username="liaison-coordinator", password="liaison-coordinator+password")) r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) @@ -715,7 +736,7 @@ def test_add_incoming_liaison(self): from_groups = [ str(g.pk) for g in Group.objects.filter(type="sdo") ] to_group = Group.objects.get(acronym="mars") submitter = Person.objects.get(user__username="marschairman") - today = date_today(datetime.timezone.utc) + today = date_today(datetime.UTC) related_liaison = liaison r = self.client.post(url, dict(from_groups=from_groups, @@ -740,7 +761,7 @@ def test_add_incoming_liaison(self): l = LiaisonStatement.objects.all().order_by("-id")[0] self.assertEqual(l.from_groups.count(),2) - self.assertEqual(l.from_contact.address, submitter.email_address()) + self.assertEqual(l.from_contact, submitter.email_address()) self.assertSequenceEqual(l.to_groups.all(),[to_group]) self.assertEqual(l.technical_contacts, "technical_contact@example.com") self.assertEqual(l.action_holder_contacts, "action_holder_contacts@example.com") @@ -775,8 +796,11 @@ def test_add_incoming_liaison(self): self.assertTrue("Liaison Statement" in outbox[-1]["Subject"]) self.assertTrue('to_contacts@' in outbox[-1]['To']) + self.assertTrue(submitter.email_address(), outbox[-1]['To']) self.assertTrue('cc@' in outbox[-1]['Cc']) + + def test_add_outgoing_liaison(self): RoleFactory(name_id='liaiman',group__type_id='sdo', person__user__username='ulm-liaiman') wg = RoleFactory(name_id='chair',person__user__username='marschairman',group__acronym='mars').group @@ -800,7 +824,7 @@ def test_add_outgoing_liaison(self): from_group = Group.objects.get(acronym="mars") to_group = Group.objects.filter(type="sdo")[0] submitter = Person.objects.get(user__username="marschairman") - today = date_today(datetime.timezone.utc) + today = date_today(datetime.UTC) related_liaison = liaison r = self.client.post(url, dict(from_groups=str(from_group.pk), @@ -825,7 +849,7 @@ def test_add_outgoing_liaison(self): l = LiaisonStatement.objects.all().order_by("-id")[0] self.assertSequenceEqual(l.from_groups.all(), [from_group]) - self.assertEqual(l.from_contact.address, submitter.email_address()) + self.assertEqual(l.from_contact, submitter.email_address()) self.assertSequenceEqual(l.to_groups.all(), [to_group]) self.assertEqual(l.to_contacts, "to_contacts@example.com") self.assertEqual(l.technical_contacts, "technical_contact@example.com") @@ -856,37 +880,7 @@ def test_add_outgoing_liaison(self): self.assertEqual(len(outbox), mailbox_before + 1) self.assertTrue("Liaison Statement" in outbox[-1]["Subject"]) self.assertTrue('aread@' in outbox[-1]['To']) - - def test_add_outgoing_liaison_unapproved_post_only(self): - RoleFactory(name_id='liaiman',group__type_id='sdo', person__user__username='ulm-liaiman') - mars = RoleFactory(name_id='chair',person__user__username='marschairman',group__acronym='mars').group - RoleFactory(name_id='ad',group=mars) - - url = urlreverse('ietf.liaisons.views.liaison_add', kwargs={'type':'outgoing'}) - login_testing_unauthorized(self, "secretary", url) - - # add new - mailbox_before = len(outbox) - from_group = Group.objects.get(acronym="mars") - to_group = Group.objects.filter(type="sdo")[0] - submitter = Person.objects.get(user__username="marschairman") - today = date_today(datetime.timezone.utc) - r = self.client.post(url, - dict(from_groups=str(from_group.pk), - from_contact=submitter.email_address(), - to_groups=str(to_group.pk), - to_contacts='to_contacts@example.com', - approved="", - purpose="info", - title="title", - submitted_date=today.strftime("%Y-%m-%d"), - body="body", - post_only="1", - )) - self.assertEqual(r.status_code, 302) - l = LiaisonStatement.objects.all().order_by("-id")[0] - self.assertEqual(l.state.slug,'pending') - self.assertEqual(len(outbox), mailbox_before + 1) + self.assertTrue(submitter.email_address(), outbox[-1]['Cc']) def test_liaison_add_attachment(self): liaison = LiaisonStatementFactory(deadline=date_today(DEADLINE_TZINFO)+datetime.timedelta(days=1)) @@ -901,7 +895,7 @@ def test_liaison_add_attachment(self): file.name = "upload.txt" post_data = dict( from_groups = ','.join([ str(x.pk) for x in liaison.from_groups.all() ]), - from_contact = liaison.from_contact.address, + from_contact = liaison.from_contact, to_groups = ','.join([ str(x.pk) for x in liaison.to_groups.all() ]), to_contacts = 'to_contacts@example.com', purpose = liaison.purpose.slug, @@ -931,26 +925,43 @@ def test_liaison_add_attachment(self): ) def test_liaison_edit_attachment(self): - - attachment = LiaisonStatementAttachmentFactory(document__name='liaiatt-1') - url = urlreverse('ietf.liaisons.views.liaison_edit_attachment', kwargs=dict(object_id=attachment.statement_id,doc_id=attachment.document_id)) + attachment = LiaisonStatementAttachmentFactory(document__name="liaiatt-1") + url = urlreverse( + "ietf.liaisons.views.liaison_edit_attachment", + kwargs=dict( + object_id=attachment.statement_id, doc_id=attachment.document_id + ), + ) login_testing_unauthorized(self, "secretary", url) r = self.client.get(url) self.assertEqual(r.status_code, 200) - post_data = dict(title='New Title') - r = self.client.post(url,post_data) + post_data = dict(title="New Title") + r = self.client.post(url, post_data) attachment = LiaisonStatementAttachment.objects.get(pk=attachment.pk) self.assertEqual(r.status_code, 302) - self.assertEqual(attachment.document.title,'New Title') - - def test_liaison_delete_attachment(self): - attachment = LiaisonStatementAttachmentFactory(document__name='liaiatt-1') - liaison = attachment.statement - url = urlreverse('ietf.liaisons.views.liaison_delete_attachment', kwargs=dict(object_id=liaison.pk,attach_id=attachment.pk)) - login_testing_unauthorized(self, "secretary", url) + self.assertEqual(attachment.document.title, "New Title") + + # ensure attempts to edit attachments not attached to this liaison statement fail + other_attachment = LiaisonStatementAttachmentFactory(document__name="liaiatt-2") + url = urlreverse( + "ietf.liaisons.views.liaison_edit_attachment", + kwargs=dict( + object_id=attachment.statement_id, doc_id=other_attachment.document_id + ), + ) r = self.client.get(url) - self.assertEqual(r.status_code, 302) - self.assertEqual(liaison.liaisonstatementattachment_set.filter(removed=False).count(),0) + self.assertEqual(r.status_code, 404) + r = self.client.post(url, dict(title="New Title")) + self.assertEqual(r.status_code, 404) + + # def test_liaison_delete_attachment(self): + # attachment = LiaisonStatementAttachmentFactory(document__name='liaiatt-1') + # liaison = attachment.statement + # url = urlreverse('ietf.liaisons.views.liaison_delete_attachment', kwargs=dict(object_id=liaison.pk,attach_id=attachment.pk)) + # login_testing_unauthorized(self, "secretary", url) + # r = self.client.get(url) + # self.assertEqual(r.status_code, 302) + # self.assertEqual(liaison.liaisonstatementattachment_set.filter(removed=False).count(),0) def test_in_response(self): '''A statement with purpose=in_response must have related statement specified''' @@ -1054,7 +1065,7 @@ def test_search(self): LiaisonStatementEventFactory(type_id='posted', statement__body="Has recently in its body",statement__from_groups=[GroupFactory(type_id='sdo',acronym='ulm'),]) # Statement 2 s2 = LiaisonStatementEventFactory(type_id='posted', statement__body="That word does not occur here", statement__title="Nor does it occur here") - s2.time=datetime.datetime(2010, 1, 1, tzinfo=datetime.timezone.utc) + s2.time=datetime.datetime(2010, 1, 1, tzinfo=datetime.UTC) s2.save() # test list only, no search filters @@ -1119,17 +1130,6 @@ def test_redirect_for_approval(self): # ------------------------------------------------- # Form validations # ------------------------------------------------- - def test_post_and_send_fail(self): - RoleFactory(name_id='liaiman',person__user__username='ulm-liaiman',group__type_id='sdo',group__acronym='ulm') - GroupFactory(type_id='wg',acronym='mars') - - url = urlreverse('ietf.liaisons.views.liaison_add', kwargs={'type':'incoming'}) - login_testing_unauthorized(self, "ulm-liaiman", url) - - r = self.client.post(url,get_liaison_post_data(),follow=True) - - self.assertEqual(r.status_code, 200) - self.assertContains(r, 'As an IETF Liaison Manager you can not send incoming liaison statements') def test_deadline_field(self): '''Required for action, comment, not info, response''' diff --git a/ietf/liaisons/tests_forms.py b/ietf/liaisons/tests_forms.py new file mode 100644 index 0000000000..101c0c8298 --- /dev/null +++ b/ietf/liaisons/tests_forms.py @@ -0,0 +1,217 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +from ietf.group.factories import GroupFactory, RoleFactory +from ietf.group.models import Group +from ietf.liaisons.forms import ( + flatten_choices, + choices_from_group_queryset, + all_internal_groups, + internal_groups_for_person, + external_groups_for_person, +) +from ietf.person.factories import PersonFactory +from ietf.person.models import Person +from ietf.utils.test_utils import TestCase + + +class HelperTests(TestCase): + @staticmethod + def _alphabetically_by_acronym(group_list): + return sorted(group_list, key=lambda item: item.acronym) + + def test_choices_from_group_queryset(self): + main_groups = list(Group.objects.filter(acronym__in=["ietf", "iab"])) + areas = GroupFactory.create_batch(2, type_id="area") + wgs = GroupFactory.create_batch(2) + + # No groups + self.assertEqual( + choices_from_group_queryset(Group.objects.none()), + [], + ) + + # Main groups only + choices = choices_from_group_queryset( + Group.objects.filter(pk__in=[g.pk for g in main_groups]) + ) + self.assertEqual(len(choices), 1, "show one optgroup, hide empty ones") + self.assertEqual(choices[0][0], "Main IETF Entities") + self.assertEqual( + [val for val, _ in choices[0][1]], # extract the choice value + [g.pk for g in self._alphabetically_by_acronym(main_groups)], + ) + + # Area groups only + choices = choices_from_group_queryset( + Group.objects.filter(pk__in=[g.pk for g in areas]) + ) + self.assertEqual(len(choices), 1, "show one optgroup, hide empty ones") + self.assertEqual(choices[0][0], "IETF Areas") + self.assertEqual( + [val for val, _ in choices[0][1]], # extract the choice value + [g.pk for g in self._alphabetically_by_acronym(areas)], + ) + + # WGs only + choices = choices_from_group_queryset( + Group.objects.filter(pk__in=[g.pk for g in wgs]) + ) + self.assertEqual(len(choices), 1, "show one optgroup, hide empty ones") + self.assertEqual(choices[0][0], "IETF Working Groups") + self.assertEqual( + [val for val, _ in choices[0][1]], # extract the choice value + [g.pk for g in self._alphabetically_by_acronym(wgs)], + ) + + # All together + choices = choices_from_group_queryset( + Group.objects.filter(pk__in=[g.pk for g in main_groups + areas + wgs]) + ) + self.assertEqual(len(choices), 3, "show all three optgroups") + self.assertEqual( + [optgroup_label for optgroup_label, _ in choices], + ["Main IETF Entities", "IETF Areas", "IETF Working Groups"], + ) + self.assertEqual( + [val for val, _ in choices[0][1]], # extract the choice value + [g.pk for g in self._alphabetically_by_acronym(main_groups)], + ) + self.assertEqual( + [val for val, _ in choices[1][1]], # extract the choice value + [g.pk for g in self._alphabetically_by_acronym(areas)], + ) + self.assertEqual( + [val for val, _ in choices[2][1]], # extract the choice value + [g.pk for g in self._alphabetically_by_acronym(wgs)], + ) + + def test_all_internal_groups(self): + # test relies on the data created in ietf.utils.test_data.make_immutable_test_data() + self.assertCountEqual( + all_internal_groups().values_list("acronym", flat=True), + {"ietf", "iab", "iesg", "farfut", "ops", "sops"}, + ) + + def test_internal_groups_for_person(self): + # test relies on the data created in ietf.utils.test_data.make_immutable_test_data() + # todo add liaison coordinator when modeled + RoleFactory( + name_id="auth", + group__type_id="sdo", + group__acronym="sdo", + person__user__username="sdo-authperson", + ) + + self.assertQuerysetEqual( + internal_groups_for_person(None), + Group.objects.none(), + msg="no Person means no groups", + ) + self.assertQuerysetEqual( + internal_groups_for_person(PersonFactory()), + Group.objects.none(), + msg="no Role means no groups", + ) + + for username in ( + "secretary", + "ietf-chair", + "iab-chair", + "sdo-authperson", + ): + returned_queryset = internal_groups_for_person( + Person.objects.get(user__username=username) + ) + self.assertCountEqual( + returned_queryset.values_list("acronym", flat=True), + {"ietf", "iab", "iesg", "farfut", "ops", "sops"}, + f"{username} should get all groups", + ) + + # "ops-ad" user is the AD of the "ops" area, which contains the "sops" wg + self.assertCountEqual( + internal_groups_for_person( + Person.objects.get(user__username="ops-ad") + ).values_list("acronym", flat=True), + {"ietf", "iesg", "ops", "sops"}, + "area director should get only their area, its wgs, and the ietf/iesg groups", + ) + + self.assertCountEqual( + internal_groups_for_person( + Person.objects.get(user__username="sopschairman"), + ).values_list("acronym", flat=True), + {"sops"}, + "wg chair should get only their wg", + ) + + def test_external_groups_for_person(self): + RoleFactory(name_id="liaison_coordinator", group__acronym="iab", person__user__username="liaison-coordinator") + the_sdo = GroupFactory(type_id="sdo", acronym="the-sdo") + liaison_manager = RoleFactory(name_id="liaiman", group=the_sdo).person + authperson = RoleFactory(name_id="auth", group=the_sdo).person + + GroupFactory(acronym="other-sdo", type_id="sdo") + for username in ( + "secretary", + "ietf-chair", + "iab-chair", + "liaison-coordinator", + "ad", + "sopschairman", + "sopssecretary", + ): + person = Person.objects.get(user__username=username) + self.assertCountEqual( + external_groups_for_person( + person, + ).values_list("acronym", flat=True), + {"the-sdo", "other-sdo"}, + f"{username} should get all SDO groups", + ) + tmp_role = RoleFactory(name_id="chair", group__type_id="wg", person=person) + self.assertCountEqual( + external_groups_for_person( + person, + ).values_list("acronym", flat=True), + {"the-sdo", "other-sdo"}, + f"{username} should still get all SDO groups when they also a liaison manager", + ) + tmp_role.delete() + + self.assertCountEqual( + external_groups_for_person(liaison_manager).values_list( + "acronym", flat=True + ), + {"the-sdo"}, + "liaison manager should get only their SDO group", + ) + self.assertCountEqual( + external_groups_for_person(authperson).values_list("acronym", flat=True), + {"the-sdo"}, + "authorized individual should get only their SDO group", + ) + + def test_flatten_choices(self): + self.assertEqual(flatten_choices([]), []) + self.assertEqual( + flatten_choices( + ( + ("group A", ()), + ("group B", (("val0", "label0"), ("val1", "label1"))), + ("group C", (("val2", "label2"),)), + ) + ), + [("val0", "label0"), ("val1", "label1"), ("val2", "label2")], + ) + + +class IncomingLiaisonFormTests(TestCase): + pass + + +class OutgoingLiaisonFormTests(TestCase): + pass + + +class EditLiaisonFormTests(TestCase): + pass diff --git a/ietf/liaisons/urls.py b/ietf/liaisons/urls.py index a4afbfef5d..498df3b965 100644 --- a/ietf/liaisons/urls.py +++ b/ietf/liaisons/urls.py @@ -26,8 +26,8 @@ url(r'^(?P\d+)/$', views.liaison_detail), url(r'^(?P\d+)/addcomment/$', views.add_comment), url(r'^(?P\d+)/edit/$', views.liaison_edit), - url(r'^(?P\d+)/edit-attachment/(?P[A-Za-z0-9._+-]+)$', views.liaison_edit_attachment), - url(r'^(?P\d+)/delete-attachment/(?P[A-Za-z0-9._+-]+)$', views.liaison_delete_attachment), + url(r'^(?P\d+)/edit-attachment/(?P[0-9]+)$', views.liaison_edit_attachment), + url(r'^(?P\d+)/delete-attachment/(?P[0-9]+)$', views.liaison_delete_attachment), url(r'^(?P\d+)/history/$', views.liaison_history), url(r'^(?P\d+)/reply/$', views.liaison_reply), url(r'^(?P\d+)/resend/$', views.liaison_resend), @@ -37,4 +37,5 @@ url(r'^add/$', views.redirect_add), url(r'^for_approval/$', views.redirect_for_approval), url(r'^for_approval/(?P\d+)/$', views.redirect_for_approval), + url(r"^list_other_sdo/$", views.list_other_sdo), ] diff --git a/ietf/liaisons/utils.py b/ietf/liaisons/utils.py index df48831917..469bbc5c87 100644 --- a/ietf/liaisons/utils.py +++ b/ietf/liaisons/utils.py @@ -4,6 +4,21 @@ from ietf.liaisons.models import LiaisonStatement from ietf.ietfauth.utils import has_role, passes_test_decorator +# Roles allowed to create and manage outgoing liaison statements. +OUTGOING_LIAISON_ROLES = [ + "Area Director", + "IAB Chair", + "IETF Chair", + "Liaison Manager", + "Liaison Coordinator", + "Secretariat", + "WG Chair", + "WG Secretary", +] + +# Roles allowed to create and manage incoming liaison statements. +INCOMING_LIAISON_ROLES = ["Authorized Individual", "Liaison Manager", "Liaison Coordinator", "Secretariat"] + can_submit_liaison_required = passes_test_decorator( lambda u, *args, **kwargs: can_add_liaison(u), "Restricted to participants who are authorized to submit liaison statements on behalf of the various IETF entities") @@ -30,13 +45,13 @@ def can_edit_liaison(user, liaison): '''Returns True if user has edit / approval authority. True if: - - user is Secretariat + - user is Secretariat or Liaison Coordinator - liaison is outgoing and user has approval authority - user is liaison manager of all SDOs involved ''' if not user.is_authenticated: return False - if has_role(user, "Secretariat"): + if has_role(user, "Secretariat") or has_role(user, "Liaison Coordinator"): return True if liaison.is_outgoing() and liaison in approvable_liaison_statements(user): @@ -59,11 +74,10 @@ def get_person_for_user(user): return None def can_add_outgoing_liaison(user): - return has_role(user, ["Area Director","WG Chair","WG Secretary","IETF Chair","IAB Chair", - "IAB Executive Director","Liaison Manager","Secretariat"]) + return has_role(user, OUTGOING_LIAISON_ROLES) def can_add_incoming_liaison(user): - return has_role(user, ["Liaison Manager","Authorized Individual","Secretariat"]) + return has_role(user, INCOMING_LIAISON_ROLES) def can_add_liaison(user): return can_add_incoming_liaison(user) or can_add_outgoing_liaison(user) diff --git a/ietf/liaisons/views.py b/ietf/liaisons/views.py index a8e80a5194..59c6ea69fc 100644 --- a/ietf/liaisons/views.py +++ b/ietf/liaisons/views.py @@ -7,19 +7,17 @@ from django.contrib import messages from django.urls import reverse as urlreverse -from django.core.exceptions import ValidationError +from django.core.exceptions import ValidationError, ObjectDoesNotExist, PermissionDenied from django.core.validators import validate_email from django.db.models import Q, Prefetch -from django.http import HttpResponse +from django.http import Http404, HttpResponse from django.shortcuts import render, get_object_or_404, redirect import debug # pyflakes:ignore -from ietf.doc.models import Document from ietf.ietfauth.utils import role_required, has_role from ietf.group.models import Group, Role -from ietf.liaisons.models import (LiaisonStatement,LiaisonStatementEvent, - LiaisonStatementAttachment) +from ietf.liaisons.models import LiaisonStatement,LiaisonStatementEvent from ietf.liaisons.utils import (get_person_for_user, can_add_outgoing_liaison, can_add_incoming_liaison, can_edit_liaison,can_submit_liaison_required, can_add_liaison) @@ -29,13 +27,6 @@ from ietf.name.models import LiaisonStatementTagName from ietf.utils.response import permission_denied -EMAIL_ALIASES = { - 'IETFCHAIR':'The IETF Chair ', - 'IESG':'The IESG ', - 'IAB':'The IAB ', - 'IABCHAIR':'The IAB Chair ', - 'IABEXECUTIVEDIRECTOR':'The IAB Executive Director '} - # ------------------------------------------------- # Helper Functions # ------------------------------------------------- @@ -57,7 +48,7 @@ def _can_take_care(liaison, user): return False if user.is_authenticated: - if has_role(user, "Secretariat"): + if has_role(user, "Secretariat") or has_role(user, "Liaison Coordinator"): return True else: return _find_person_in_emails(liaison, get_person_for_user(user)) @@ -84,8 +75,6 @@ def _find_person_in_emails(liaison, person): return True elif addr in ('iab@iab.org', 'iab-chair@iab.org') and has_role(person.user, "IAB Chair"): return True - elif addr in ('execd@iab.org', ) and has_role(person.user, "IAB Executive Director"): - return True return False @@ -97,66 +86,6 @@ def contacts_from_roles(roles): emails = [ contact_email_from_role(r) for r in roles ] return ','.join(emails) -def get_cc(group): - '''Returns list of emails to use as CC for group. Simplified refactor of IETFHierarchy - get_cc() and get_from_cc() - ''' - emails = [] - - # role based CCs - if group.acronym in ('ietf','iesg'): - emails.append(EMAIL_ALIASES['IESG']) - emails.append(EMAIL_ALIASES['IETFCHAIR']) - elif group.acronym in ('iab'): - emails.append(EMAIL_ALIASES['IAB']) - emails.append(EMAIL_ALIASES['IABCHAIR']) - emails.append(EMAIL_ALIASES['IABEXECUTIVEDIRECTOR']) - elif group.type_id == 'area': - emails.append(EMAIL_ALIASES['IETFCHAIR']) - ad_roles = group.role_set.filter(name='ad') - emails.extend([ contact_email_from_role(r) for r in ad_roles ]) - elif group.type_id == 'wg': - ad_roles = group.parent.role_set.filter(name='ad') - emails.extend([ contact_email_from_role(r) for r in ad_roles ]) - chair_roles = group.role_set.filter(name='chair') - emails.extend([ contact_email_from_role(r) for r in chair_roles ]) - if group.list_email: - emails.append('{} Discussion List <{}>'.format(group.name,group.list_email)) - elif group.type_id == 'sdo': - liaiman_roles = group.role_set.filter(name='liaiman') - emails.extend([ contact_email_from_role(r) for r in liaiman_roles ]) - - # explicit CCs - liaison_cc_roles = group.role_set.filter(name='liaison_cc_contact') - emails.extend([ contact_email_from_role(r) for r in liaison_cc_roles ]) - - return emails - -def get_contacts_for_group(group): - '''Returns default contacts for groups as a comma separated string''' - # use explicit default contacts if defined - explicit_contacts = contacts_from_roles(group.role_set.filter(name='liaison_contact')) - if explicit_contacts: - return explicit_contacts - - # otherwise construct based on group type - contacts = [] - if group.type_id == 'area': - roles = group.role_set.filter(name='ad') - contacts.append(contacts_from_roles(roles)) - elif group.type_id == 'wg': - roles = group.role_set.filter(name='chair') - contacts.append(contacts_from_roles(roles)) - elif group.acronym == 'ietf': - contacts.append(EMAIL_ALIASES['IETFCHAIR']) - elif group.acronym == 'iab': - contacts.append(EMAIL_ALIASES['IABCHAIR']) - contacts.append(EMAIL_ALIASES['IABEXECUTIVEDIRECTOR']) - elif group.acronym == 'iesg': - contacts.append(EMAIL_ALIASES['IESG']) - - return ','.join(contacts) - def get_details_tabs(stmt, selected): return [ t + (t[0].lower() == selected.lower(),) @@ -171,7 +100,7 @@ def needs_approval(group,person): user = person.user if group.acronym in ('ietf','iesg') and has_role(user, 'IETF Chair'): return False - if group.acronym == 'iab' and (has_role(user,'IAB Chair') or has_role(user,'IAB Executive Director')): + if group.acronym == 'iab' and has_role(user,'IAB Chair'): return False if group.type_id == 'area' and group.role_set.filter(name='ad',person=person): return False @@ -189,23 +118,14 @@ def normalize_sort(request): return sort, order_by -def post_only(group,person): - '''Returns true if the user is restricted to post_only (vs. post_and_send) for this - group. This is for incoming liaison statements. - - Secretariat have full access. - - Authorized Individuals have full access for the group they are associated with - - Liaison Managers can post only - ''' - if group.type_id == 'sdo' and ( not(has_role(person.user,"Secretariat") or group.role_set.filter(name='auth',person=person)) ): - return True - else: - return False # ------------------------------------------------- # Ajax Functions # ------------------------------------------------- @can_submit_liaison_required def ajax_get_liaison_info(request): + from ietf.mailtrigger.utils import get_contacts_for_liaison_messages_for_group_primary,get_contacts_for_liaison_messages_for_group_secondary + '''Returns dictionary of info to update entry form given the groups that have been selected ''' @@ -222,20 +142,18 @@ def ajax_get_liaison_info(request): cc = [] does_need_approval = [] - can_post_only = [] to_contacts = [] response_contacts = [] - result = {'response_contacts':[],'to_contacts': [], 'cc': [], 'needs_approval': False, 'post_only': False, 'full_list': []} + result = {'response_contacts':[],'to_contacts': [], 'cc': [], 'needs_approval': False, 'full_list': []} for group in from_groups: - cc.extend(get_cc(group)) + cc.extend(get_contacts_for_liaison_messages_for_group_primary(group)) does_need_approval.append(needs_approval(group,person)) - can_post_only.append(post_only(group,person)) - response_contacts.append(get_contacts_for_group(group)) + response_contacts.append(get_contacts_for_liaison_messages_for_group_secondary(group)) for group in to_groups: - cc.extend(get_cc(group)) - to_contacts.append(get_contacts_for_group(group)) + cc.extend(get_contacts_for_liaison_messages_for_group_primary(group)) + to_contacts.append(get_contacts_for_liaison_messages_for_group_secondary(group)) # if there are from_groups and any need approval if does_need_approval: @@ -246,12 +164,15 @@ def ajax_get_liaison_info(request): else: does_need_approval = True - result.update({'error': False, - 'cc': list(set(cc)), - 'response_contacts':list(set(response_contacts)), - 'to_contacts': list(set(to_contacts)), - 'needs_approval': does_need_approval, - 'post_only': any(can_post_only)}) + result.update( + { + "error": False, + "cc": list(set(cc)), + "response_contacts": list(set(response_contacts)), + "to_contacts": list(set(to_contacts)), + "needs_approval": does_need_approval, + } + ) json_result = json.dumps(result) return HttpResponse(json_result, content_type='application/json') @@ -375,23 +296,29 @@ def liaison_history(request, object_id): def liaison_delete_attachment(request, object_id, attach_id): liaison = get_object_or_404(LiaisonStatement, pk=object_id) - attach = get_object_or_404(LiaisonStatementAttachment, pk=attach_id) + if not can_edit_liaison(request.user, liaison): permission_denied(request, "You are not authorized for this action.") - - # FIXME: this view should use POST instead of GET when deleting - attach.removed = True - attach.save() - - # create event - LiaisonStatementEvent.objects.create( - type_id='modified', - by=get_person_for_user(request.user), - statement=liaison, - desc='Attachment Removed: {}'.format(attach.document.title) - ) - messages.success(request, 'Attachment Deleted') - return redirect('ietf.liaisons.views.liaison_detail', object_id=liaison.pk) + else: + permission_denied(request, "This operation is temporarily unavailable. Ask the secretariat to mark the attachment as removed using the admin.") + + # The following will be replaced with a different approach in the next generation of the liaison tool + # attach = get_object_or_404(LiaisonStatementAttachment, pk=attach_id) + + # # FIXME: this view should use POST instead of GET when deleting + # attach.removed = True + # debug.say("Got here") + # attach.save() + + # # create event + # LiaisonStatementEvent.objects.create( + # type_id='modified', + # by=get_person_for_user(request.user), + # statement=liaison, + # desc='Attachment Removed: {}'.format(attach.document.title) + # ) + # messages.success(request, 'Attachment Deleted') + # return redirect('ietf.liaisons.views.liaison_detail', object_id=liaison.pk) def liaison_detail(request, object_id): liaison = get_object_or_404(LiaisonStatement, pk=object_id) @@ -402,22 +329,28 @@ def liaison_detail(request, object_id): if request.method == 'POST': - if request.POST.get('approved'): - liaison.change_state(state_id='approved',person=person) - liaison.change_state(state_id='posted',person=person) - send_liaison_by_email(request, liaison) - messages.success(request,'Liaison Statement Approved and Posted') - elif request.POST.get('dead'): - liaison.change_state(state_id='dead',person=person) - messages.success(request,'Liaison Statement Killed') - elif request.POST.get('resurrect'): - liaison.change_state(state_id='pending',person=person) - messages.success(request,'Liaison Statement Resurrected') - elif request.POST.get('do_action_taken') and can_take_care: + if request.POST.get('do_action_taken') and can_take_care: liaison.tags.remove('required') liaison.tags.add('taken') can_take_care = False messages.success(request,'Action handled') + else: + if can_edit: + if request.POST.get('approved'): + liaison.change_state(state_id='approved',person=person) + liaison.change_state(state_id='posted',person=person) + send_liaison_by_email(request, liaison) + messages.success(request,'Liaison Statement Approved and Posted') + elif request.POST.get('dead'): + liaison.change_state(state_id='dead',person=person) + messages.success(request,'Liaison Statement Killed') + elif request.POST.get('resurrect'): + liaison.change_state(state_id='pending',person=person) + messages.success(request,'Liaison Statement Resurrected') + else: + pass + else: + raise PermissionDenied() relations_by = [i.target for i in liaison.source_of_set.filter(target__state__slug='posted')] relations_to = [i.source for i in liaison.target_of_set.filter(source__state__slug='posted')] @@ -441,7 +374,11 @@ def liaison_edit(request, object_id): def liaison_edit_attachment(request, object_id, doc_id): '''Edit the Liaison Statement attachment title''' liaison = get_object_or_404(LiaisonStatement, pk=object_id) - doc = get_object_or_404(Document, pk=doc_id) + try: + doc = liaison.attachments.get(pk=doc_id) + except ObjectDoesNotExist: + raise Http404 + if not can_edit_liaison(request.user, liaison): permission_denied(request, "You are not authorized for this action.") @@ -572,3 +509,17 @@ def liaison_resend(request, object_id): messages.success(request,'Liaison Statement resent') return redirect('ietf.liaisons.views.liaison_list') + +@role_required("Secretariat", "IAB", "Liaison Coordinator", "Liaison Manager") +def list_other_sdo(request): + def _sdo_order_key(obj:Group)-> tuple[str,str]: + state_order = { + "active" : "a", + "conclude": "b", + } + return (state_order.get(obj.state.slug,f"c{obj.state.slug}"), obj.acronym) + + sdos = sorted(list(Group.objects.filter(type="sdo")),key = _sdo_order_key) + for sdo in sdos: + sdo.liaison_managers =[r.person for r in sdo.role_set.filter(name="liaiman")] + return render(request,"liaisons/list_other_sdo.html",dict(sdos=sdos)) diff --git a/ietf/liaisons/widgets.py b/ietf/liaisons/widgets.py index 74368e83f2..48db8af0a3 100644 --- a/ietf/liaisons/widgets.py +++ b/ietf/liaisons/widgets.py @@ -26,7 +26,9 @@ def render(self, name, value, **kwargs): html += '%s' % conditional_escape(i) required_str = 'Please fill in %s to attach a new file' % conditional_escape(self.required_label) html += '%s' % conditional_escape(required_str) - html += '' % conditional_escape(self.label) + html += ''.format( + f"id_{name}", conditional_escape(self.label) + ) return mark_safe(html) diff --git a/ietf/mailtrigger/admin.py b/ietf/mailtrigger/admin.py index a60fd5b072..8c73f2ae02 100644 --- a/ietf/mailtrigger/admin.py +++ b/ietf/mailtrigger/admin.py @@ -1,9 +1,10 @@ -# Copyright The IETF Trust 2015-2019, All Rights Reserved +# Copyright The IETF Trust 2015-2025, All Rights Reserved from django.contrib import admin +from simple_history.admin import SimpleHistoryAdmin from ietf.mailtrigger.models import MailTrigger, Recipient -class RecipientAdmin(admin.ModelAdmin): +class RecipientAdmin(SimpleHistoryAdmin): list_display = [ 'slug', 'desc', 'template', 'has_code', ] def has_code(self, obj): return hasattr(obj,'gather_%s'%obj.slug) @@ -11,7 +12,7 @@ def has_code(self, obj): admin.site.register(Recipient, RecipientAdmin) -class MailTriggerAdmin(admin.ModelAdmin): +class MailTriggerAdmin(SimpleHistoryAdmin): list_display = [ 'slug', 'desc', ] filter_horizontal = [ 'to', 'cc', ] admin.site.register(MailTrigger, MailTriggerAdmin) diff --git a/ietf/mailtrigger/forms.py b/ietf/mailtrigger/forms.py index 366c429d8c..8d13c5edf3 100644 --- a/ietf/mailtrigger/forms.py +++ b/ietf/mailtrigger/forms.py @@ -11,6 +11,7 @@ class CcSelectForm(forms.Form): expansions = dict() # type: Dict[str, List[str]] cc_choices = forms.MultipleChoiceField( + required=False, label='Cc', choices=[], widget=forms.CheckboxSelectMultiple(), diff --git a/ietf/mailtrigger/migrations/0006_call_for_adoption_and_last_call_issued.py b/ietf/mailtrigger/migrations/0006_call_for_adoption_and_last_call_issued.py new file mode 100644 index 0000000000..7adad150eb --- /dev/null +++ b/ietf/mailtrigger/migrations/0006_call_for_adoption_and_last_call_issued.py @@ -0,0 +1,43 @@ +# Copyright The IETF Trust 2023, All Rights Reserved + +from django.db import migrations + + +def forward(apps, schema_editor): + MailTrigger = apps.get_model("mailtrigger", "MailTrigger") + Recipient = apps.get_model("mailtrigger", "Recipient") + recipients = list( + Recipient.objects.filter( + slug__in=( + "doc_group_mail_list", + "doc_authors", + "doc_group_chairs", + "doc_shepherd", + ) + ) + ) + call_for_adoption = MailTrigger.objects.create( + slug="doc_wg_call_for_adoption_issued", + desc="Recipients when a working group call for adoption is issued", + ) + call_for_adoption.to.add(*recipients) + wg_last_call = MailTrigger.objects.create( + slug="doc_wg_last_call_issued", + desc="Recipients when a working group last call is issued", + ) + wg_last_call.to.add(*recipients) + + +def reverse(apps, schema_editor): + MailTrigger = apps.get_model("mailtrigger", "MailTrigger") + MailTrigger.objects.filter( + slug_in=("doc_wg_call_for_adoption_issued", "doc_wg_last_call_issued") + ).delete() + + +class Migration(migrations.Migration): + dependencies = [ + ("mailtrigger", "0005_rfc_recipients"), + ] + + operations = [migrations.RunPython(forward, reverse)] diff --git a/ietf/mailtrigger/migrations/0007_historicalrecipient_historicalmailtrigger.py b/ietf/mailtrigger/migrations/0007_historicalrecipient_historicalmailtrigger.py new file mode 100644 index 0000000000..d23b72d737 --- /dev/null +++ b/ietf/mailtrigger/migrations/0007_historicalrecipient_historicalmailtrigger.py @@ -0,0 +1,122 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +from io import StringIO + +from django.conf import settings +from django.core import management +from django.db import migrations, models +import django.db.models.deletion +import simple_history.models + +from ietf.utils.log import log + + +def forward(apps, schema_editor): + # Fill in history for existing data using the populate_history management command + captured_stdout = StringIO() + captured_stderr = StringIO() + try: + management.call_command( + "populate_history", + "mailtrigger.MailTrigger", + "mailtrigger.Recipient", + stdout=captured_stdout, + stderr=captured_stderr, + ) + except management.CommandError as err: + log( + "Failed to populate history for mailtrigger models.\n" + "\n" + f"stdout:\n{captured_stdout.getvalue() or ''}\n" + "\n" + f"stderr:\n{captured_stderr.getvalue() or ''}\n" + ) + raise RuntimeError("Failed to populate history for mailtrigger models") from err + log( + "Populated history for mailtrigger models.\n" + "\n" + f"stdout:\n{captured_stdout.getvalue() or ''}\n" + "\n" + f"stderr:\n{captured_stderr.getvalue() or ''}\n" + ) + + +def reverse(apps, schema_editor): + pass # nothing to do + + +class Migration(migrations.Migration): + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("mailtrigger", "0006_call_for_adoption_and_last_call_issued"), + ] + + operations = [ + migrations.CreateModel( + name="HistoricalRecipient", + fields=[ + ("slug", models.CharField(db_index=True, max_length=32)), + ("desc", models.TextField(blank=True)), + ("template", models.TextField(blank=True, null=True)), + ("history_id", models.AutoField(primary_key=True, serialize=False)), + ("history_date", models.DateTimeField(db_index=True)), + ("history_change_reason", models.CharField(max_length=100, null=True)), + ( + "history_type", + models.CharField( + choices=[("+", "Created"), ("~", "Changed"), ("-", "Deleted")], + max_length=1, + ), + ), + ( + "history_user", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "verbose_name": "historical recipient", + "verbose_name_plural": "historical recipients", + "ordering": ("-history_date", "-history_id"), + "get_latest_by": ("history_date", "history_id"), + }, + bases=(simple_history.models.HistoricalChanges, models.Model), + ), + migrations.CreateModel( + name="HistoricalMailTrigger", + fields=[ + ("slug", models.CharField(db_index=True, max_length=64)), + ("desc", models.TextField(blank=True)), + ("history_id", models.AutoField(primary_key=True, serialize=False)), + ("history_date", models.DateTimeField(db_index=True)), + ("history_change_reason", models.CharField(max_length=100, null=True)), + ( + "history_type", + models.CharField( + choices=[("+", "Created"), ("~", "Changed"), ("-", "Deleted")], + max_length=1, + ), + ), + ( + "history_user", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "verbose_name": "historical mail trigger", + "verbose_name_plural": "historical mail triggers", + "ordering": ("-history_date", "-history_id"), + "get_latest_by": ("history_date", "history_id"), + }, + bases=(simple_history.models.HistoricalChanges, models.Model), + ), + migrations.RunPython(forward, reverse), + ] diff --git a/ietf/mailtrigger/migrations/0008_liaison_statement_incoming_and_outgoing_posted.py b/ietf/mailtrigger/migrations/0008_liaison_statement_incoming_and_outgoing_posted.py new file mode 100644 index 0000000000..189a783a2e --- /dev/null +++ b/ietf/mailtrigger/migrations/0008_liaison_statement_incoming_and_outgoing_posted.py @@ -0,0 +1,72 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations + + +def forward(apps, schema_editor): + Mailtrigger = apps.get_model("mailtrigger", "MailTrigger") + Recipient = apps.get_model("mailtrigger", "Recipient") + recipients_to = Recipient.objects.get(pk="liaison_to_contacts") + recipients_cc = list( + Recipient.objects.filter( + slug__in=( + "liaison_cc", + "liaison_coordinators", + "liaison_response_contacts", + "liaison_technical_contacts", + ) + ) + ) + recipient_from = Recipient.objects.get(pk="liaison_from_contact") + + liaison_posted_outgoing = Mailtrigger.objects.create( + slug="liaison_statement_posted_outgoing", + desc="Recipients for a message when a new outgoing liaison statement is posted", + ) + liaison_posted_outgoing.to.add(recipients_to) + liaison_posted_outgoing.cc.add(*recipients_cc) + liaison_posted_outgoing.cc.add(recipient_from) + + liaison_posted_incoming = Mailtrigger.objects.create( + slug="liaison_statement_posted_incoming", + desc="Recipients for a message when a new incoming liaison statement is posted", + ) + liaison_posted_incoming.to.add(recipients_to) + liaison_posted_incoming.cc.add(*recipients_cc) + + Mailtrigger.objects.filter(slug=("liaison_statement_posted")).delete() + + +def reverse(apps, schema_editor): + Mailtrigger = apps.get_model("mailtrigger", "MailTrigger") + Recipient = apps.get_model("mailtrigger", "Recipient") + + Mailtrigger.objects.filter( + slug__in=( + "liaison_statement_posted_outgoing", + "liaison_statement_posted_incoming", + ) + ).delete() + + liaison_statement_posted = Mailtrigger.objects.create( + slug="liaison_statement_posted", + desc="Recipients for a message when a new liaison statement is posted", + ) + + liaison_to_contacts = Recipient.objects.get(slug="liaison_to_contacts") + recipients_ccs = Recipient.objects.filter( + slug__in=( + "liaison_cc", + "liaison_coordinators", + "liaison_response_contacts", + "liaison_technical_contacts", + ) + ) + liaison_statement_posted.to.add(liaison_to_contacts) + liaison_statement_posted.cc.add(*recipients_ccs) + + +class Migration(migrations.Migration): + dependencies = [("mailtrigger", "0007_historicalrecipient_historicalmailtrigger")] + + operations = [migrations.RunPython(forward, reverse)] diff --git a/ietf/mailtrigger/models.py b/ietf/mailtrigger/models.py index 66b7139fa5..435729f893 100644 --- a/ietf/mailtrigger/models.py +++ b/ietf/mailtrigger/models.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2015-2020, All Rights Reserved +# Copyright The IETF Trust 2015-2025, All Rights Reserved # -*- coding: utf-8 -*- @@ -7,6 +7,8 @@ from email.utils import parseaddr +from simple_history.models import HistoricalRecords + from ietf.doc.utils_bofreq import bofreq_editors, bofreq_responsible from ietf.utils.mail import formataddr, get_email_addresses_from_text from ietf.group.models import Group, Role @@ -38,6 +40,7 @@ class MailTrigger(models.Model): desc = models.TextField(blank=True) to = models.ManyToManyField('mailtrigger.Recipient', blank=True, related_name='used_in_to') cc = models.ManyToManyField('mailtrigger.Recipient', blank=True, related_name='used_in_cc') + history = HistoricalRecords() class Meta: ordering = ["slug"] @@ -49,6 +52,7 @@ class Recipient(models.Model): slug = models.CharField(max_length=32, primary_key=True) desc = models.TextField(blank=True) template = models.TextField(null=True, blank=True) + history = HistoricalRecords() class Meta: ordering = ["slug"] diff --git a/ietf/mailtrigger/resources.py b/ietf/mailtrigger/resources.py index eb5466618a..daca055bf4 100644 --- a/ietf/mailtrigger/resources.py +++ b/ietf/mailtrigger/resources.py @@ -7,7 +7,7 @@ from ietf import api -from ietf.mailtrigger.models import Recipient, MailTrigger +from ietf.mailtrigger.models import MailTrigger, Recipient class RecipientResource(ModelResource): @@ -37,3 +37,43 @@ class Meta: } api.mailtrigger.register(MailTriggerResource()) +from ietf.utils.resources import UserResource +class HistoricalMailTriggerResource(ModelResource): + history_user = ToOneField(UserResource, 'history_user', null=True) + class Meta: + queryset = MailTrigger.history.model.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'historicalmailtrigger' + ordering = ['history_id', ] + filtering = { + "slug": ALL, + "desc": ALL, + "history_id": ALL, + "history_date": ALL, + "history_change_reason": ALL, + "history_type": ALL, + "history_user": ALL_WITH_RELATIONS, + } +api.mailtrigger.register(HistoricalMailTriggerResource()) + +from ietf.utils.resources import UserResource +class HistoricalRecipientResource(ModelResource): + history_user = ToOneField(UserResource, 'history_user', null=True) + class Meta: + queryset = Recipient.history.model.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'historicalrecipient' + ordering = ['history_id', ] + filtering = { + "slug": ALL, + "desc": ALL, + "template": ALL, + "history_id": ALL, + "history_date": ALL, + "history_change_reason": ALL, + "history_type": ALL, + "history_user": ALL_WITH_RELATIONS, + } +api.mailtrigger.register(HistoricalRecipientResource()) diff --git a/ietf/mailtrigger/utils.py b/ietf/mailtrigger/utils.py index 9915eae3fd..bcdaf5e44e 100644 --- a/ietf/mailtrigger/utils.py +++ b/ietf/mailtrigger/utils.py @@ -9,6 +9,14 @@ from ietf.utils.mail import excludeaddrs +EMAIL_ALIASES = { + "IETFCHAIR": "The IETF Chair ", + "IESG": "The IESG ", + "IAB": "The IAB ", + "IABCHAIR": "The IAB Chair ", +} + + class AddrLists(namedtuple("AddrLists", ["to", "cc"])): __slots__ = () @@ -66,6 +74,69 @@ def get_mailtrigger(slug, create_from_slug_if_not_exists, desc_if_not_exists): return mailtrigger +def get_contacts_for_liaison_messages_for_group_primary(group): + from ietf.liaisons.views import contact_email_from_role + + '''Returns list of emails to use in liaison message for group + ''' + emails = [] + + # role based emails + if group.acronym in ('ietf','iesg'): + emails.append(EMAIL_ALIASES['IESG']) + emails.append(EMAIL_ALIASES['IETFCHAIR']) + elif group.acronym in ('iab'): + emails.append(EMAIL_ALIASES['IAB']) + emails.append(EMAIL_ALIASES['IABCHAIR']) + elif group.type_id == 'area': + emails.append(EMAIL_ALIASES['IETFCHAIR']) + ad_roles = group.role_set.filter(name='ad') + emails.extend([ contact_email_from_role(r) for r in ad_roles ]) + elif group.type_id == 'wg': + ad_roles = group.parent.role_set.filter(name='ad') + emails.extend([ contact_email_from_role(r) for r in ad_roles ]) + chair_roles = group.role_set.filter(name='chair') + emails.extend([ contact_email_from_role(r) for r in chair_roles ]) + if group.list_email: + emails.append('{} Discussion List <{}>'.format(group.name,group.list_email)) + elif group.type_id == 'sdo': + liaiman_roles = group.role_set.filter(name='liaiman') + emails.extend([ contact_email_from_role(r) for r in liaiman_roles ]) + + # explicit CCs + liaison_cc_roles = group.role_set.filter(name='liaison_cc_contact') + emails.extend([ contact_email_from_role(r) for r in liaison_cc_roles ]) + + return emails + + +def get_contacts_for_liaison_messages_for_group_secondary(group): + from ietf.liaisons.views import contacts_from_roles + + '''Returns default contacts for groups as a comma separated string''' + # use explicit default contacts if defined + explicit_contacts = contacts_from_roles(group.role_set.filter(name='liaison_contact')) + if explicit_contacts: + return explicit_contacts + + # otherwise construct based on group type + contacts = [] + if group.type_id == 'area': + roles = group.role_set.filter(name='ad') + contacts.append(contacts_from_roles(roles)) + elif group.type_id == 'wg': + roles = group.role_set.filter(name='chair') + contacts.append(contacts_from_roles(roles)) + elif group.acronym == 'ietf': + contacts.append(EMAIL_ALIASES['IETFCHAIR']) + elif group.acronym == 'iab': + contacts.append(EMAIL_ALIASES['IABCHAIR']) + elif group.acronym == 'iesg': + contacts.append(EMAIL_ALIASES['IESG']) + + return ','.join(contacts) + + def gather_relevant_expansions(**kwargs): def starts_with(prefix): return MailTrigger.objects.filter(slug__startswith=prefix).values_list( diff --git a/ietf/meeting/admin.py b/ietf/meeting/admin.py index b7e56c7b77..03abf5c029 100644 --- a/ietf/meeting/admin.py +++ b/ietf/meeting/admin.py @@ -3,11 +3,14 @@ from django.contrib import admin +from django.db.models import Count from ietf.meeting.models import (Attended, Meeting, Room, Session, TimeSlot, Constraint, Schedule, SchedTimeSessAssignment, ResourceAssociation, FloorPlan, UrlResource, SessionPresentation, ImportantDate, SlideSubmission, SchedulingEvent, BusinessConstraint, - ProceedingsMaterial, MeetingHost) + ProceedingsMaterial, MeetingHost, Registration, RegistrationTicket, + AttendanceTypeName) +from ietf.utils.admin import SaferTabularInline class UrlResourceAdmin(admin.ModelAdmin): @@ -16,7 +19,7 @@ class UrlResourceAdmin(admin.ModelAdmin): raw_id_fields = ['room', ] admin.site.register(UrlResource, UrlResourceAdmin) -class UrlResourceInline(admin.TabularInline): +class UrlResourceInline(SaferTabularInline): model = UrlResource class RoomAdmin(admin.ModelAdmin): @@ -26,7 +29,7 @@ class RoomAdmin(admin.ModelAdmin): admin.site.register(Room, RoomAdmin) -class RoomInline(admin.TabularInline): +class RoomInline(SaferTabularInline): model = Room class MeetingAdmin(admin.ModelAdmin): @@ -91,7 +94,7 @@ def name_lower(self, instance): admin.site.register(Constraint, ConstraintAdmin) -class SchedulingEventInline(admin.TabularInline): +class SchedulingEventInline(SaferTabularInline): model = SchedulingEvent raw_id_fields = ["by"] @@ -213,3 +216,76 @@ class AttendedAdmin(admin.ModelAdmin): search_fields = ["person__name", "session__group__acronym", "session__meeting__number", "session__name", "session__purpose__name"] raw_id_fields= ["person", "session"] admin.site.register(Attended, AttendedAdmin) + +class MeetingFilter(admin.SimpleListFilter): + title = 'Meeting Filter' + parameter_name = 'meeting_id' + + def lookups(self, request, model_admin): + # only include meetings with registration records + meetings = Meeting.objects.filter(type='ietf').annotate(reg_count=Count('registration')).filter(reg_count__gt=0).order_by('-date') + choices = meetings.values_list('id', 'number') + return choices + + def queryset(self, request, queryset): + if self.value(): + return queryset.filter(meeting__id=self.value()) + return queryset + +class AttendanceFilter(admin.SimpleListFilter): + title = 'Attendance Type' + parameter_name = 'attendance_type' + + def lookups(self, request, model_admin): + choices = AttendanceTypeName.objects.all().values_list('slug', 'name') + return choices + + def queryset(self, request, queryset): + if self.value(): + return queryset.filter(tickets__attendance_type__slug=self.value()).distinct() + return queryset + +class RegistrationTicketInline(SaferTabularInline): + model = RegistrationTicket + +class RegistrationAdmin(admin.ModelAdmin): + model = Registration + list_filter = [AttendanceFilter, MeetingFilter] + list_display = ['meeting', 'first_name', 'last_name', 'display_attendance', 'affiliation', 'country_code', 'email', ] + search_fields = ['first_name', 'last_name', 'affiliation', 'country_code', 'email', ] + raw_id_fields = ['person'] + inlines = [RegistrationTicketInline, ] + ordering = ['-meeting__date', 'last_name'] + + def display_attendance(self, instance): + '''Only display the most significant ticket in the list. + To see all the tickets inspect the individual instance + ''' + if instance.tickets.filter(attendance_type__slug='onsite').exists(): + return 'onsite' + elif instance.tickets.filter(attendance_type__slug='remote').exists(): + return 'remote' + elif instance.tickets.filter(attendance_type__slug='hackathon_onsite').exists(): + return 'hackathon onsite' + elif instance.tickets.filter(attendance_type__slug='hackathon_remote').exists(): + return 'hackathon remote' + display_attendance.short_description = "Attendance" # type: ignore # https://github.com/python/mypy/issues/2087 + +admin.site.register(Registration, RegistrationAdmin) + +class RegistrationTicketAdmin(admin.ModelAdmin): + model = RegistrationTicket + list_filter = ['attendance_type', ] + # not available until Django 5.2, the name of a related field, using the __ notation + # list_display = ['registration__meeting', 'registration', 'attendance_type', 'ticket_type', 'registration__email'] + # list_select_related = ('registration',) + list_display = ['registration', 'attendance_type', 'ticket_type', 'display_meeting'] + search_fields = ['registration__first_name', 'registration__last_name', 'registration__email'] + raw_id_fields = ['registration'] + ordering = ['-registration__meeting__date', 'registration__last_name'] + + def display_meeting(self, instance): + return instance.registration.meeting.number + display_meeting.short_description = "Meeting" # type: ignore # https://github.com/python/mypy/issues/2087 + +admin.site.register(RegistrationTicket, RegistrationTicketAdmin) diff --git a/ietf/meeting/factories.py b/ietf/meeting/factories.py index eb36e9e756..fc0ce8387c 100644 --- a/ietf/meeting/factories.py +++ b/ietf/meeting/factories.py @@ -12,7 +12,7 @@ from ietf.doc.storage_utils import store_str from ietf.meeting.models import (Attended, Meeting, Session, SchedulingEvent, Schedule, TimeSlot, SessionPresentation, FloorPlan, Room, SlideSubmission, Constraint, - MeetingHost, ProceedingsMaterial) + MeetingHost, ProceedingsMaterial, Registration, RegistrationTicket) from ietf.name.models import (ConstraintName, SessionStatusName, ProceedingsMaterialTypeName, TimerangeName, SessionPurposeName) from ietf.doc.factories import ProceedingsMaterialDocFactory @@ -318,3 +318,48 @@ class Meta: session = factory.SubFactory(SessionFactory) person = factory.SubFactory(PersonFactory) + + +class RegistrationFactory(factory.django.DjangoModelFactory): + """ + This will create an associated onsite week_pass ticket by default. + Methods of calling: + + RegistrationFactory() create a ticket with defaults, onsite + RegistrationFactory(with_ticket=True) same as above + RegistrationFactory(with_ticket={'attendance_type_id': 'remote'}) creates ticket with overrides + RegistrationFactory(with_ticket=False) does not create a ticket + """ + class Meta: + model = Registration + skip_postgeneration_save = True + + meeting = factory.SubFactory(MeetingFactory) + person = factory.SubFactory(PersonFactory) + email = factory.LazyAttribute(lambda obj: obj.person.email()) + first_name = factory.LazyAttribute(lambda obj: obj.person.first_name()) + last_name = factory.LazyAttribute(lambda obj: obj.person.last_name()) + affiliation = factory.Faker('company') + country_code = factory.Faker('country_code') + attended = False + checkedin = False + + @factory.post_generation + def with_ticket(self, create, extracted, **kwargs): + if not create: + return + if extracted is False: + # Explicitly disable ticket creation + return + ticket_kwargs = extracted if isinstance(extracted, dict) else {} + RegistrationTicketFactory(registration=self, **ticket_kwargs) + + +class RegistrationTicketFactory(factory.django.DjangoModelFactory): + class Meta: + model = RegistrationTicket + skip_postgeneration_save = True + + registration = factory.SubFactory(RegistrationFactory) + attendance_type_id = factory.LazyAttribute(lambda _: 'onsite') + ticket_type_id = factory.LazyAttribute(lambda _: 'week_pass') diff --git a/ietf/meeting/forms.py b/ietf/meeting/forms.py index e1d1e90b8d..e5b1697f86 100644 --- a/ietf/meeting/forms.py +++ b/ietf/meeting/forms.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2016-2023, All Rights Reserved +# Copyright The IETF Trust 2016-2025, All Rights Reserved # -*- coding: utf-8 -*- @@ -15,19 +15,24 @@ from django.core import validators from django.core.exceptions import ValidationError from django.forms import BaseInlineFormSet +from django.template.defaultfilters import pluralize from django.utils.functional import cached_property +from django.utils.safestring import mark_safe import debug # pyflakes:ignore from ietf.doc.models import Document, State, NewRevisionDocEvent from ietf.group.models import Group from ietf.group.utils import groups_managed_by -from ietf.meeting.models import Session, Meeting, Schedule, countries, timezones, TimeSlot, Room +from ietf.meeting.models import (Session, Meeting, Schedule, COUNTRIES, TIMEZONES, TimeSlot, Room, + Constraint, ResourceAssociation) from ietf.meeting.helpers import get_next_interim_number, make_materials_directories from ietf.meeting.helpers import is_interim_meeting_approved, get_next_agenda_name from ietf.message.models import Message -from ietf.name.models import TimeSlotTypeName, SessionPurposeName +from ietf.name.models import TimeSlotTypeName, SessionPurposeName, TimerangeName, ConstraintName +from ietf.person.fields import SearchablePersonsField from ietf.person.models import Person +from ietf.utils import log from ietf.utils.fields import ( DatepickerDateField, DatepickerSplitDateTimeWidget, @@ -35,13 +40,13 @@ ModelMultipleChoiceField, MultiEmailField, ) +from ietf.utils.html import clean_text_field from ietf.utils.validators import ( validate_file_size, validate_mime_type, validate_file_extension, validate_no_html_frame) -# need to insert empty option for use in ChoiceField -# countries.insert(0, ('', '-'*9 )) -countries.insert(0, ('', '-' * 9)) -timezones.insert(0, ('', '-' * 9)) +NUM_SESSION_CHOICES = (('', '--Please select'), ('1', '1'), ('2', '2')) +SESSION_TIME_RELATION_CHOICES = (('', 'No preference'),) + Constraint.TIME_RELATION_CHOICES +JOINT_FOR_SESSION_CHOICES = (('1', 'First session'), ('2', 'Second session'), ('3', 'Third session'), ) # ------------------------------------------------- # Helpers @@ -79,6 +84,27 @@ def duration_string(duration): return string +def allowed_conflicting_groups(): + return Group.objects.filter( + type__in=['wg', 'ag', 'rg', 'rag', 'program', 'edwg'], + state__in=['bof', 'proposed', 'active']) + + +def check_conflict(groups, source_group): + ''' + Takes a string which is a list of group acronyms. Checks that they are all active groups + ''' + # convert to python list (allow space or comma separated lists) + items = groups.replace(',', ' ').split() + active_groups = allowed_conflicting_groups() + for group in items: + if group == source_group.acronym: + raise forms.ValidationError("Cannot declare a conflict with the same group: %s" % group) + + if not active_groups.filter(acronym=group): + raise forms.ValidationError("Invalid or inactive group acronym: %s" % group) + + # ------------------------------------------------- # Forms # ------------------------------------------------- @@ -140,12 +166,12 @@ class InterimMeetingModelForm(forms.ModelForm): approved = forms.BooleanField(required=False) city = forms.CharField(max_length=255, required=False) city.widget.attrs['placeholder'] = "City" - country = forms.ChoiceField(choices=countries, required=False) + country = forms.ChoiceField(choices=COUNTRIES, required=False) country.widget.attrs['class'] = "select2-field" country.widget.attrs['data-max-entries'] = 1 country.widget.attrs['data-placeholder'] = "Country" country.widget.attrs['data-minimum-input-length'] = 0 - time_zone = forms.ChoiceField(choices=timezones) + time_zone = forms.ChoiceField(choices=TIMEZONES) time_zone.widget.attrs['class'] = "select2-field" time_zone.widget.attrs['data-max-entries'] = 1 time_zone.widget.attrs['data-minimum-input-length'] = 0 @@ -758,6 +784,9 @@ def __init__(self, group, *args, **kwargs): self.fields['purpose'].queryset = SessionPurposeName.objects.filter(pk__in=session_purposes) if not group.features.acts_like_wg: self.fields['requested_duration'].durations = [datetime.timedelta(minutes=m) for m in range(30, 241, 30)] + # add bootstrap classes + self.fields['purpose'].widget.attrs.update({'class': 'form-select'}) + self.fields['type'].widget.attrs.update({'class': 'form-select', 'aria-label': 'session type'}) class Meta: model = Session @@ -842,3 +871,296 @@ def sessiondetailsformset_factory(min_num=1, max_num=3): max_num=max_num, extra=max_num, # only creates up to max_num total ) + + +class SessionRequestStatusForm(forms.Form): + message = forms.CharField(widget=forms.Textarea(attrs={'rows': '3', 'cols': '80'}), strip=False) + + +class NameModelMultipleChoiceField(ModelMultipleChoiceField): + def label_from_instance(self, name): + return name.desc + + +class SessionRequestForm(forms.Form): + num_session = forms.ChoiceField( + choices=NUM_SESSION_CHOICES, + label="Number of sessions") + # session fields are added in __init__() + session_time_relation = forms.ChoiceField( + choices=SESSION_TIME_RELATION_CHOICES, + required=False, + label="Time between two sessions") + attendees = forms.IntegerField(label="Number of Attendees") + # FIXME: it would cleaner to have these be + # ModelMultipleChoiceField, and just customize the widgetry, that + # way validation comes for free (applies to this CharField and the + # constraints dynamically instantiated in __init__()) + joint_with_groups = forms.CharField(max_length=255, required=False) + joint_with_groups_selector = forms.ChoiceField(choices=[], required=False) # group select widget for prev field + joint_for_session = forms.ChoiceField(choices=JOINT_FOR_SESSION_CHOICES, required=False) + comments = forms.CharField( + max_length=200, + label='Special Requests', + help_text='i.e. restrictions on meeting times / days, etc. (limit 200 characters)', + required=False) + third_session = forms.BooleanField( + required=False, + help_text="Help") + resources = forms.MultipleChoiceField( + widget=forms.CheckboxSelectMultiple, + required=False, + label='Resources Requested') + bethere = SearchablePersonsField( + label="Participants who must be present", + required=False, + help_text=mark_safe('Do not include Area Directors and WG Chairs; the system already tracks their availability.')) + timeranges = NameModelMultipleChoiceField( + widget=forms.CheckboxSelectMultiple, + required=False, + label=mark_safe('Times during which this WG can not meet:
Please explain any selections in Special Requests below.'), + queryset=TimerangeName.objects.all()) + adjacent_with_wg = forms.ChoiceField( + required=False, + label=mark_safe('Plan session adjacent with another WG:
(Immediately before or after another WG, no break in between, in the same room.)')) + send_notifications = forms.BooleanField(label="Send notification emails?", required=False, initial=False) + + def __init__(self, group, meeting, data=None, *args, **kwargs): + self.hidden = kwargs.pop('hidden', False) + self.notifications_optional = kwargs.pop('notifications_optional', False) + + self.group = group + formset_class = sessiondetailsformset_factory(max_num=3 if group.features.acts_like_wg else 50) + self.session_forms = formset_class(group=self.group, meeting=meeting, data=data) + super().__init__(data=data, *args, **kwargs) + if not self.notifications_optional: + self.fields['send_notifications'].widget = forms.HiddenInput() + + # Allow additional sessions for non-wg-like groups + if not self.group.features.acts_like_wg: + self.fields['num_session'].choices = ((n, str(n)) for n in range(1, 51)) + + self._add_widget_class(self.fields['third_session'].widget, 'form-check-input') + self.fields['comments'].widget = forms.Textarea(attrs={'rows': '3', 'cols': '65'}) + + other_groups = list(allowed_conflicting_groups().exclude(pk=group.pk).values_list('acronym', 'acronym').order_by('acronym')) + self.fields['adjacent_with_wg'].choices = [('', '--No preference')] + other_groups + group_acronym_choices = [('', '--Select WG(s)')] + other_groups + self.fields['joint_with_groups_selector'].choices = group_acronym_choices + + # Set up constraints for the meeting + self._wg_field_data = [] + for constraintname in meeting.group_conflict_types.all(): + # two fields for each constraint: a CharField for the group list and a selector to add entries + constraint_field = forms.CharField(max_length=255, required=False) + constraint_field.widget.attrs['data-slug'] = constraintname.slug + constraint_field.widget.attrs['data-constraint-name'] = str(constraintname).title() + constraint_field.widget.attrs['aria-label'] = f'{constraintname.slug}_input' + self._add_widget_class(constraint_field.widget, 'wg_constraint') + self._add_widget_class(constraint_field.widget, 'form-control') + + selector_field = forms.ChoiceField(choices=group_acronym_choices, required=False) + selector_field.widget.attrs['data-slug'] = constraintname.slug # used by onchange handler + self._add_widget_class(selector_field.widget, 'wg_constraint_selector') + self._add_widget_class(selector_field.widget, 'form-control') + + cfield_id = 'constraint_{}'.format(constraintname.slug) + cselector_id = 'wg_selector_{}'.format(constraintname.slug) + # keep an eye out for field name conflicts + log.assertion('cfield_id not in self.fields') + log.assertion('cselector_id not in self.fields') + self.fields[cfield_id] = constraint_field + self.fields[cselector_id] = selector_field + self._wg_field_data.append((constraintname, cfield_id, cselector_id)) + + # Show constraints that are not actually used by the meeting so these don't get lost + self._inactive_wg_field_data = [] + inactive_cnames = ConstraintName.objects.filter( + is_group_conflict=True # Only collect group conflicts... + ).exclude( + meeting=meeting # ...that are not enabled for this meeting... + ).filter( + constraint__source=group, # ...but exist for this group... + constraint__meeting=meeting, # ... at this meeting. + ).distinct() + + for inactive_constraint_name in inactive_cnames: + field_id = 'delete_{}'.format(inactive_constraint_name.slug) + self.fields[field_id] = forms.BooleanField(required=False, label='Delete this conflict', help_text='Delete this inactive conflict?') + self._add_widget_class(self.fields[field_id].widget, 'form-control') + constraints = group.constraint_source_set.filter(meeting=meeting, name=inactive_constraint_name) + self._inactive_wg_field_data.append( + (inactive_constraint_name, + ' '.join([c.target.acronym for c in constraints]), + field_id) + ) + + self.fields['joint_with_groups_selector'].widget.attrs['onchange'] = "document.form_post.joint_with_groups.value=document.form_post.joint_with_groups.value + ' ' + this.options[this.selectedIndex].value; return 1;" + self.fields["resources"].choices = [(x.pk, x.desc) for x in ResourceAssociation.objects.filter(name__used=True).order_by('name__order')] + + if self.hidden: + # replace all the widgets to start... + for key in list(self.fields.keys()): + self.fields[key].widget = forms.HiddenInput() + # re-replace a couple special cases + self.fields['resources'].widget = forms.MultipleHiddenInput() + self.fields['timeranges'].widget = forms.MultipleHiddenInput() + # and entirely replace bethere - no need to support searching if input is hidden + self.fields['bethere'] = ModelMultipleChoiceField( + widget=forms.MultipleHiddenInput, required=False, + queryset=Person.objects.all(), + ) + + def wg_constraint_fields(self): + """Iterates over wg constraint fields + + Intended for use in the template. + """ + for cname, cfield_id, cselector_id in self._wg_field_data: + yield cname, self[cfield_id], self[cselector_id] + + def wg_constraint_count(self): + """How many wg constraints are there?""" + return len(self._wg_field_data) + + def wg_constraint_field_ids(self): + """Iterates over wg constraint field IDs""" + for cname, cfield_id, _ in self._wg_field_data: + yield cname, cfield_id + + def inactive_wg_constraints(self): + for cname, value, field_id in self._inactive_wg_field_data: + yield cname, value, self[field_id] + + def inactive_wg_constraint_count(self): + return len(self._inactive_wg_field_data) + + def inactive_wg_constraint_field_ids(self): + """Iterates over wg constraint field IDs""" + for cname, _, field_id in self._inactive_wg_field_data: + yield cname, field_id + + @staticmethod + def _add_widget_class(widget, new_class): + """Add a new class, taking care in case some already exist""" + existing_classes = widget.attrs.get('class', '').split() + widget.attrs['class'] = ' '.join(existing_classes + [new_class]) + + def _join_conflicts(self, cleaned_data, slugs): + """Concatenate constraint fields from cleaned data into a single list""" + conflicts = [] + for cname, cfield_id, _ in self._wg_field_data: + if cname.slug in slugs and cfield_id in cleaned_data: + groups = cleaned_data[cfield_id] + # convert to python list (allow space or comma separated lists) + items = groups.replace(',', ' ').split() + conflicts.extend(items) + return conflicts + + def _validate_duplicate_conflicts(self, cleaned_data): + """Validate that no WGs appear in more than one constraint that does not allow duplicates + + Raises ValidationError + """ + # Only the older constraints (conflict, conflic2, conflic3) need to be mutually exclusive. + all_conflicts = self._join_conflicts(cleaned_data, ['conflict', 'conflic2', 'conflic3']) + seen = [] + duplicated = [] + errors = [] + for c in all_conflicts: + if c not in seen: + seen.append(c) + elif c not in duplicated: # only report once + duplicated.append(c) + errors.append(forms.ValidationError('%s appears in conflicts more than once' % c)) + return errors + + def clean_joint_with_groups(self): + groups = self.cleaned_data['joint_with_groups'] + check_conflict(groups, self.group) + return groups + + def clean_comments(self): + return clean_text_field(self.cleaned_data['comments']) + + def clean_bethere(self): + bethere = self.cleaned_data["bethere"] + if bethere: + extra = set( + Person.objects.filter( + role__group=self.group, role__name__in=["chair", "ad"] + ) + & bethere + ) + if extra: + extras = ", ".join(e.name for e in extra) + raise forms.ValidationError( + ( + f"Please remove the following person{pluralize(len(extra))}, the system " + f"tracks their availability due to their role{pluralize(len(extra))}: {extras}." + ) + ) + return bethere + + def clean_send_notifications(self): + return True if not self.notifications_optional else self.cleaned_data['send_notifications'] + + def is_valid(self): + return super().is_valid() and self.session_forms.is_valid() + + def clean(self): + super(SessionRequestForm, self).clean() + self.session_forms.clean() + + data = self.cleaned_data + + # Validate the individual conflict fields + for _, cfield_id, _ in self._wg_field_data: + try: + check_conflict(data[cfield_id], self.group) + except forms.ValidationError as e: + self.add_error(cfield_id, e) + + # Skip remaining tests if individual field tests had errors, + if self.errors: + return data + + # error if conflicts contain disallowed dupes + for error in self._validate_duplicate_conflicts(data): + self.add_error(None, error) + + # Verify expected number of session entries are present + num_sessions_with_data = len(self.session_forms.forms_to_keep) + num_sessions_expected = -1 + try: + num_sessions_expected = int(data.get('num_session', '')) + except ValueError: + self.add_error('num_session', 'Invalid value for number of sessions') + if num_sessions_with_data < num_sessions_expected: + self.add_error('num_session', 'Must provide data for all sessions') + + # if default (empty) option is selected, cleaned_data won't include num_session key + if num_sessions_expected != 2 and num_sessions_expected is not None: + if data.get('session_time_relation'): + self.add_error( + 'session_time_relation', + forms.ValidationError('Time between sessions can only be used when two sessions are requested.') + ) + + joint_session = data.get('joint_for_session', '') + if joint_session != '': + joint_session = int(joint_session) + if joint_session > num_sessions_with_data: + self.add_error( + 'joint_for_session', + forms.ValidationError( + f'Session {joint_session} can not be the joint session, the session has not been requested.' + ) + ) + + return data + + @property + def media(self): + # get media for our formset + return super().media + self.session_forms.media + forms.Media(js=('ietf/js/session_form.js',)) diff --git a/ietf/meeting/migrations/0012_registration_registrationticket.py b/ietf/meeting/migrations/0012_registration_registrationticket.py new file mode 100644 index 0000000000..c555f52e8b --- /dev/null +++ b/ietf/meeting/migrations/0012_registration_registrationticket.py @@ -0,0 +1,90 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations, models +import django.db.models.deletion +import ietf.utils.models + + +class Migration(migrations.Migration): + + dependencies = [ + ("name", "0017_populate_new_reg_names"), + ("person", "0004_alter_person_photo_alter_person_photo_thumb"), + ("meeting", "0011_alter_slidesubmission_doc"), + ] + + operations = [ + migrations.CreateModel( + name="Registration", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("first_name", models.CharField(max_length=255)), + ("last_name", models.CharField(max_length=255)), + ("affiliation", models.CharField(blank=True, max_length=255)), + ("country_code", models.CharField(max_length=2)), + ("email", models.EmailField(blank=True, max_length=254, null=True)), + ("attended", models.BooleanField(default=False)), + ("checkedin", models.BooleanField(default=False)), + ( + "meeting", + ietf.utils.models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="meeting.meeting", + ), + ), + ( + "person", + ietf.utils.models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.PROTECT, + to="person.person", + ), + ), + ], + ), + migrations.CreateModel( + name="RegistrationTicket", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "attendance_type", + ietf.utils.models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + to="name.attendancetypename", + ), + ), + ( + "registration", + ietf.utils.models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="tickets", + to="meeting.registration", + ), + ), + ( + "ticket_type", + ietf.utils.models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + to="name.registrationtickettypename", + ), + ), + ], + ), + ] diff --git a/ietf/meeting/migrations/0013_correct_reg_checkedin.py b/ietf/meeting/migrations/0013_correct_reg_checkedin.py new file mode 100644 index 0000000000..88b3efceac --- /dev/null +++ b/ietf/meeting/migrations/0013_correct_reg_checkedin.py @@ -0,0 +1,41 @@ +# Generated by Django 4.2.21 on 2025-05-20 22:28 + +''' +The original migration had a flaw. If a participant had both a remote and onsite +registration, which is rare but does occur, which registration the checkedin state +came from was indeterminate. If it came from the remote registration it would be +False which might be wrong. This migration finds all registrations with onsite tickets +and checkedin is False, and checks if it is correct, and fixes if needed. +''' + +from django.db import migrations +import datetime + + +def forward(apps, schema_editor): + Registration = apps.get_model('meeting', 'Registration') + MeetingRegistration = apps.get_model('stats', 'MeetingRegistration') + today = datetime.date.today() + for reg in Registration.objects.filter(tickets__attendance_type__slug='onsite', checkedin=False, meeting__date__lt=today).order_by('meeting__number'): + # get original MeetingRegistration + mregs = MeetingRegistration.objects.filter(meeting=reg.meeting, email=reg.email, reg_type='onsite') + mregs_checkedin = [mr.checkedin for mr in mregs] + if any(mregs_checkedin): + reg.checkedin = True + reg.save() + print(f'updating {reg.meeting}:{reg.email}:{reg.pk}') + + +def reverse(apps, schema_editor): + pass + + +class Migration(migrations.Migration): + + dependencies = [ + ("meeting", "0012_registration_registrationticket"), + ] + + operations = [ + migrations.RunPython(forward, reverse), + ] diff --git a/ietf/meeting/migrations/0014_alter_floorplan_image.py b/ietf/meeting/migrations/0014_alter_floorplan_image.py new file mode 100644 index 0000000000..e125625edc --- /dev/null +++ b/ietf/meeting/migrations/0014_alter_floorplan_image.py @@ -0,0 +1,25 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations, models +import ietf.meeting.models +import ietf.utils.storage + + +class Migration(migrations.Migration): + dependencies = [ + ("meeting", "0013_correct_reg_checkedin"), + ] + + operations = [ + migrations.AlterField( + model_name="floorplan", + name="image", + field=models.ImageField( + default=None, + storage=ietf.utils.storage.BlobShadowFileSystemStorage( + kind="", location=None + ), + upload_to=ietf.meeting.models.floorplan_path, + ), + ), + ] diff --git a/ietf/meeting/migrations/0015_alter_meeting_time_zone.py b/ietf/meeting/migrations/0015_alter_meeting_time_zone.py new file mode 100644 index 0000000000..2a4b7859ee --- /dev/null +++ b/ietf/meeting/migrations/0015_alter_meeting_time_zone.py @@ -0,0 +1,451 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations, models + + +def forward(apps, schema_editor): + """Migrate 'GMT' meeting time zones to 'UTC'""" + Meeting = apps.get_model("meeting", "Meeting") + Meeting.objects.filter(time_zone="GMT").update(time_zone="UTC") + + +def reverse(apps, schema_editor): + pass # nothing to do + + +class Migration(migrations.Migration): + + dependencies = [ + ("meeting", "0014_alter_floorplan_image"), + ] + + operations = [ + migrations.RunPython(forward, reverse), + migrations.AlterField( + model_name="meeting", + name="time_zone", + field=models.CharField( + choices=[ + ("", "---------"), + ("Africa/Abidjan", "Africa/Abidjan"), + ("Africa/Accra", "Africa/Accra"), + ("Africa/Addis_Ababa", "Africa/Addis_Ababa"), + ("Africa/Algiers", "Africa/Algiers"), + ("Africa/Asmara", "Africa/Asmara"), + ("Africa/Bamako", "Africa/Bamako"), + ("Africa/Bangui", "Africa/Bangui"), + ("Africa/Banjul", "Africa/Banjul"), + ("Africa/Bissau", "Africa/Bissau"), + ("Africa/Blantyre", "Africa/Blantyre"), + ("Africa/Brazzaville", "Africa/Brazzaville"), + ("Africa/Bujumbura", "Africa/Bujumbura"), + ("Africa/Cairo", "Africa/Cairo"), + ("Africa/Casablanca", "Africa/Casablanca"), + ("Africa/Ceuta", "Africa/Ceuta"), + ("Africa/Conakry", "Africa/Conakry"), + ("Africa/Dakar", "Africa/Dakar"), + ("Africa/Dar_es_Salaam", "Africa/Dar_es_Salaam"), + ("Africa/Djibouti", "Africa/Djibouti"), + ("Africa/Douala", "Africa/Douala"), + ("Africa/El_Aaiun", "Africa/El_Aaiun"), + ("Africa/Freetown", "Africa/Freetown"), + ("Africa/Gaborone", "Africa/Gaborone"), + ("Africa/Harare", "Africa/Harare"), + ("Africa/Johannesburg", "Africa/Johannesburg"), + ("Africa/Juba", "Africa/Juba"), + ("Africa/Kampala", "Africa/Kampala"), + ("Africa/Khartoum", "Africa/Khartoum"), + ("Africa/Kigali", "Africa/Kigali"), + ("Africa/Kinshasa", "Africa/Kinshasa"), + ("Africa/Lagos", "Africa/Lagos"), + ("Africa/Libreville", "Africa/Libreville"), + ("Africa/Lome", "Africa/Lome"), + ("Africa/Luanda", "Africa/Luanda"), + ("Africa/Lubumbashi", "Africa/Lubumbashi"), + ("Africa/Lusaka", "Africa/Lusaka"), + ("Africa/Malabo", "Africa/Malabo"), + ("Africa/Maputo", "Africa/Maputo"), + ("Africa/Maseru", "Africa/Maseru"), + ("Africa/Mbabane", "Africa/Mbabane"), + ("Africa/Mogadishu", "Africa/Mogadishu"), + ("Africa/Monrovia", "Africa/Monrovia"), + ("Africa/Nairobi", "Africa/Nairobi"), + ("Africa/Ndjamena", "Africa/Ndjamena"), + ("Africa/Niamey", "Africa/Niamey"), + ("Africa/Nouakchott", "Africa/Nouakchott"), + ("Africa/Ouagadougou", "Africa/Ouagadougou"), + ("Africa/Porto-Novo", "Africa/Porto-Novo"), + ("Africa/Sao_Tome", "Africa/Sao_Tome"), + ("Africa/Tripoli", "Africa/Tripoli"), + ("Africa/Tunis", "Africa/Tunis"), + ("Africa/Windhoek", "Africa/Windhoek"), + ("America/Adak", "America/Adak"), + ("America/Anchorage", "America/Anchorage"), + ("America/Anguilla", "America/Anguilla"), + ("America/Antigua", "America/Antigua"), + ("America/Araguaina", "America/Araguaina"), + ( + "America/Argentina/Buenos_Aires", + "America/Argentina/Buenos_Aires", + ), + ("America/Argentina/Catamarca", "America/Argentina/Catamarca"), + ("America/Argentina/Cordoba", "America/Argentina/Cordoba"), + ("America/Argentina/Jujuy", "America/Argentina/Jujuy"), + ("America/Argentina/La_Rioja", "America/Argentina/La_Rioja"), + ("America/Argentina/Mendoza", "America/Argentina/Mendoza"), + ( + "America/Argentina/Rio_Gallegos", + "America/Argentina/Rio_Gallegos", + ), + ("America/Argentina/Salta", "America/Argentina/Salta"), + ("America/Argentina/San_Juan", "America/Argentina/San_Juan"), + ("America/Argentina/San_Luis", "America/Argentina/San_Luis"), + ("America/Argentina/Tucuman", "America/Argentina/Tucuman"), + ("America/Argentina/Ushuaia", "America/Argentina/Ushuaia"), + ("America/Aruba", "America/Aruba"), + ("America/Asuncion", "America/Asuncion"), + ("America/Atikokan", "America/Atikokan"), + ("America/Bahia", "America/Bahia"), + ("America/Bahia_Banderas", "America/Bahia_Banderas"), + ("America/Barbados", "America/Barbados"), + ("America/Belem", "America/Belem"), + ("America/Belize", "America/Belize"), + ("America/Blanc-Sablon", "America/Blanc-Sablon"), + ("America/Boa_Vista", "America/Boa_Vista"), + ("America/Bogota", "America/Bogota"), + ("America/Boise", "America/Boise"), + ("America/Cambridge_Bay", "America/Cambridge_Bay"), + ("America/Campo_Grande", "America/Campo_Grande"), + ("America/Cancun", "America/Cancun"), + ("America/Caracas", "America/Caracas"), + ("America/Cayenne", "America/Cayenne"), + ("America/Cayman", "America/Cayman"), + ("America/Chicago", "America/Chicago"), + ("America/Chihuahua", "America/Chihuahua"), + ("America/Costa_Rica", "America/Costa_Rica"), + ("America/Creston", "America/Creston"), + ("America/Cuiaba", "America/Cuiaba"), + ("America/Curacao", "America/Curacao"), + ("America/Danmarkshavn", "America/Danmarkshavn"), + ("America/Dawson", "America/Dawson"), + ("America/Dawson_Creek", "America/Dawson_Creek"), + ("America/Denver", "America/Denver"), + ("America/Detroit", "America/Detroit"), + ("America/Dominica", "America/Dominica"), + ("America/Edmonton", "America/Edmonton"), + ("America/Eirunepe", "America/Eirunepe"), + ("America/El_Salvador", "America/El_Salvador"), + ("America/Fort_Nelson", "America/Fort_Nelson"), + ("America/Fortaleza", "America/Fortaleza"), + ("America/Glace_Bay", "America/Glace_Bay"), + ("America/Goose_Bay", "America/Goose_Bay"), + ("America/Grand_Turk", "America/Grand_Turk"), + ("America/Grenada", "America/Grenada"), + ("America/Guadeloupe", "America/Guadeloupe"), + ("America/Guatemala", "America/Guatemala"), + ("America/Guayaquil", "America/Guayaquil"), + ("America/Guyana", "America/Guyana"), + ("America/Halifax", "America/Halifax"), + ("America/Havana", "America/Havana"), + ("America/Hermosillo", "America/Hermosillo"), + ("America/Indiana/Indianapolis", "America/Indiana/Indianapolis"), + ("America/Indiana/Knox", "America/Indiana/Knox"), + ("America/Indiana/Marengo", "America/Indiana/Marengo"), + ("America/Indiana/Petersburg", "America/Indiana/Petersburg"), + ("America/Indiana/Tell_City", "America/Indiana/Tell_City"), + ("America/Indiana/Vevay", "America/Indiana/Vevay"), + ("America/Indiana/Vincennes", "America/Indiana/Vincennes"), + ("America/Indiana/Winamac", "America/Indiana/Winamac"), + ("America/Inuvik", "America/Inuvik"), + ("America/Iqaluit", "America/Iqaluit"), + ("America/Jamaica", "America/Jamaica"), + ("America/Juneau", "America/Juneau"), + ("America/Kentucky/Louisville", "America/Kentucky/Louisville"), + ("America/Kentucky/Monticello", "America/Kentucky/Monticello"), + ("America/La_Paz", "America/La_Paz"), + ("America/Lima", "America/Lima"), + ("America/Los_Angeles", "America/Los_Angeles"), + ("America/Maceio", "America/Maceio"), + ("America/Managua", "America/Managua"), + ("America/Manaus", "America/Manaus"), + ("America/Martinique", "America/Martinique"), + ("America/Matamoros", "America/Matamoros"), + ("America/Mazatlan", "America/Mazatlan"), + ("America/Menominee", "America/Menominee"), + ("America/Merida", "America/Merida"), + ("America/Metlakatla", "America/Metlakatla"), + ("America/Mexico_City", "America/Mexico_City"), + ("America/Miquelon", "America/Miquelon"), + ("America/Moncton", "America/Moncton"), + ("America/Monterrey", "America/Monterrey"), + ("America/Montevideo", "America/Montevideo"), + ("America/Montserrat", "America/Montserrat"), + ("America/Nassau", "America/Nassau"), + ("America/New_York", "America/New_York"), + ("America/Nipigon", "America/Nipigon"), + ("America/Nome", "America/Nome"), + ("America/Noronha", "America/Noronha"), + ("America/North_Dakota/Beulah", "America/North_Dakota/Beulah"), + ("America/North_Dakota/Center", "America/North_Dakota/Center"), + ( + "America/North_Dakota/New_Salem", + "America/North_Dakota/New_Salem", + ), + ("America/Nuuk", "America/Nuuk"), + ("America/Ojinaga", "America/Ojinaga"), + ("America/Panama", "America/Panama"), + ("America/Pangnirtung", "America/Pangnirtung"), + ("America/Paramaribo", "America/Paramaribo"), + ("America/Phoenix", "America/Phoenix"), + ("America/Port-au-Prince", "America/Port-au-Prince"), + ("America/Port_of_Spain", "America/Port_of_Spain"), + ("America/Porto_Velho", "America/Porto_Velho"), + ("America/Puerto_Rico", "America/Puerto_Rico"), + ("America/Punta_Arenas", "America/Punta_Arenas"), + ("America/Rainy_River", "America/Rainy_River"), + ("America/Rankin_Inlet", "America/Rankin_Inlet"), + ("America/Recife", "America/Recife"), + ("America/Regina", "America/Regina"), + ("America/Resolute", "America/Resolute"), + ("America/Rio_Branco", "America/Rio_Branco"), + ("America/Santarem", "America/Santarem"), + ("America/Santiago", "America/Santiago"), + ("America/Santo_Domingo", "America/Santo_Domingo"), + ("America/Sao_Paulo", "America/Sao_Paulo"), + ("America/Scoresbysund", "America/Scoresbysund"), + ("America/Sitka", "America/Sitka"), + ("America/St_Johns", "America/St_Johns"), + ("America/St_Kitts", "America/St_Kitts"), + ("America/St_Lucia", "America/St_Lucia"), + ("America/St_Thomas", "America/St_Thomas"), + ("America/St_Vincent", "America/St_Vincent"), + ("America/Swift_Current", "America/Swift_Current"), + ("America/Tegucigalpa", "America/Tegucigalpa"), + ("America/Thule", "America/Thule"), + ("America/Thunder_Bay", "America/Thunder_Bay"), + ("America/Tijuana", "America/Tijuana"), + ("America/Toronto", "America/Toronto"), + ("America/Tortola", "America/Tortola"), + ("America/Vancouver", "America/Vancouver"), + ("America/Whitehorse", "America/Whitehorse"), + ("America/Winnipeg", "America/Winnipeg"), + ("America/Yakutat", "America/Yakutat"), + ("America/Yellowknife", "America/Yellowknife"), + ("Antarctica/Casey", "Antarctica/Casey"), + ("Antarctica/Davis", "Antarctica/Davis"), + ("Antarctica/DumontDUrville", "Antarctica/DumontDUrville"), + ("Antarctica/Macquarie", "Antarctica/Macquarie"), + ("Antarctica/Mawson", "Antarctica/Mawson"), + ("Antarctica/McMurdo", "Antarctica/McMurdo"), + ("Antarctica/Palmer", "Antarctica/Palmer"), + ("Antarctica/Rothera", "Antarctica/Rothera"), + ("Antarctica/Syowa", "Antarctica/Syowa"), + ("Antarctica/Troll", "Antarctica/Troll"), + ("Antarctica/Vostok", "Antarctica/Vostok"), + ("Asia/Aden", "Asia/Aden"), + ("Asia/Almaty", "Asia/Almaty"), + ("Asia/Amman", "Asia/Amman"), + ("Asia/Anadyr", "Asia/Anadyr"), + ("Asia/Aqtau", "Asia/Aqtau"), + ("Asia/Aqtobe", "Asia/Aqtobe"), + ("Asia/Ashgabat", "Asia/Ashgabat"), + ("Asia/Atyrau", "Asia/Atyrau"), + ("Asia/Baghdad", "Asia/Baghdad"), + ("Asia/Bahrain", "Asia/Bahrain"), + ("Asia/Baku", "Asia/Baku"), + ("Asia/Bangkok", "Asia/Bangkok"), + ("Asia/Barnaul", "Asia/Barnaul"), + ("Asia/Beirut", "Asia/Beirut"), + ("Asia/Bishkek", "Asia/Bishkek"), + ("Asia/Brunei", "Asia/Brunei"), + ("Asia/Chita", "Asia/Chita"), + ("Asia/Choibalsan", "Asia/Choibalsan"), + ("Asia/Colombo", "Asia/Colombo"), + ("Asia/Damascus", "Asia/Damascus"), + ("Asia/Dhaka", "Asia/Dhaka"), + ("Asia/Dili", "Asia/Dili"), + ("Asia/Dubai", "Asia/Dubai"), + ("Asia/Dushanbe", "Asia/Dushanbe"), + ("Asia/Famagusta", "Asia/Famagusta"), + ("Asia/Gaza", "Asia/Gaza"), + ("Asia/Hebron", "Asia/Hebron"), + ("Asia/Ho_Chi_Minh", "Asia/Ho_Chi_Minh"), + ("Asia/Hong_Kong", "Asia/Hong_Kong"), + ("Asia/Hovd", "Asia/Hovd"), + ("Asia/Irkutsk", "Asia/Irkutsk"), + ("Asia/Jakarta", "Asia/Jakarta"), + ("Asia/Jayapura", "Asia/Jayapura"), + ("Asia/Jerusalem", "Asia/Jerusalem"), + ("Asia/Kabul", "Asia/Kabul"), + ("Asia/Kamchatka", "Asia/Kamchatka"), + ("Asia/Karachi", "Asia/Karachi"), + ("Asia/Kathmandu", "Asia/Kathmandu"), + ("Asia/Khandyga", "Asia/Khandyga"), + ("Asia/Kolkata", "Asia/Kolkata"), + ("Asia/Krasnoyarsk", "Asia/Krasnoyarsk"), + ("Asia/Kuala_Lumpur", "Asia/Kuala_Lumpur"), + ("Asia/Kuching", "Asia/Kuching"), + ("Asia/Kuwait", "Asia/Kuwait"), + ("Asia/Macau", "Asia/Macau"), + ("Asia/Magadan", "Asia/Magadan"), + ("Asia/Makassar", "Asia/Makassar"), + ("Asia/Manila", "Asia/Manila"), + ("Asia/Muscat", "Asia/Muscat"), + ("Asia/Nicosia", "Asia/Nicosia"), + ("Asia/Novokuznetsk", "Asia/Novokuznetsk"), + ("Asia/Novosibirsk", "Asia/Novosibirsk"), + ("Asia/Omsk", "Asia/Omsk"), + ("Asia/Oral", "Asia/Oral"), + ("Asia/Phnom_Penh", "Asia/Phnom_Penh"), + ("Asia/Pontianak", "Asia/Pontianak"), + ("Asia/Pyongyang", "Asia/Pyongyang"), + ("Asia/Qatar", "Asia/Qatar"), + ("Asia/Qostanay", "Asia/Qostanay"), + ("Asia/Qyzylorda", "Asia/Qyzylorda"), + ("Asia/Riyadh", "Asia/Riyadh"), + ("Asia/Sakhalin", "Asia/Sakhalin"), + ("Asia/Samarkand", "Asia/Samarkand"), + ("Asia/Seoul", "Asia/Seoul"), + ("Asia/Shanghai", "Asia/Shanghai"), + ("Asia/Singapore", "Asia/Singapore"), + ("Asia/Srednekolymsk", "Asia/Srednekolymsk"), + ("Asia/Taipei", "Asia/Taipei"), + ("Asia/Tashkent", "Asia/Tashkent"), + ("Asia/Tbilisi", "Asia/Tbilisi"), + ("Asia/Tehran", "Asia/Tehran"), + ("Asia/Thimphu", "Asia/Thimphu"), + ("Asia/Tokyo", "Asia/Tokyo"), + ("Asia/Tomsk", "Asia/Tomsk"), + ("Asia/Ulaanbaatar", "Asia/Ulaanbaatar"), + ("Asia/Urumqi", "Asia/Urumqi"), + ("Asia/Ust-Nera", "Asia/Ust-Nera"), + ("Asia/Vientiane", "Asia/Vientiane"), + ("Asia/Vladivostok", "Asia/Vladivostok"), + ("Asia/Yakutsk", "Asia/Yakutsk"), + ("Asia/Yangon", "Asia/Yangon"), + ("Asia/Yekaterinburg", "Asia/Yekaterinburg"), + ("Asia/Yerevan", "Asia/Yerevan"), + ("Atlantic/Azores", "Atlantic/Azores"), + ("Atlantic/Bermuda", "Atlantic/Bermuda"), + ("Atlantic/Canary", "Atlantic/Canary"), + ("Atlantic/Cape_Verde", "Atlantic/Cape_Verde"), + ("Atlantic/Faroe", "Atlantic/Faroe"), + ("Atlantic/Madeira", "Atlantic/Madeira"), + ("Atlantic/Reykjavik", "Atlantic/Reykjavik"), + ("Atlantic/South_Georgia", "Atlantic/South_Georgia"), + ("Atlantic/St_Helena", "Atlantic/St_Helena"), + ("Atlantic/Stanley", "Atlantic/Stanley"), + ("Australia/Adelaide", "Australia/Adelaide"), + ("Australia/Brisbane", "Australia/Brisbane"), + ("Australia/Broken_Hill", "Australia/Broken_Hill"), + ("Australia/Darwin", "Australia/Darwin"), + ("Australia/Eucla", "Australia/Eucla"), + ("Australia/Hobart", "Australia/Hobart"), + ("Australia/Lindeman", "Australia/Lindeman"), + ("Australia/Lord_Howe", "Australia/Lord_Howe"), + ("Australia/Melbourne", "Australia/Melbourne"), + ("Australia/Perth", "Australia/Perth"), + ("Australia/Sydney", "Australia/Sydney"), + ("Europe/Amsterdam", "Europe/Amsterdam"), + ("Europe/Andorra", "Europe/Andorra"), + ("Europe/Astrakhan", "Europe/Astrakhan"), + ("Europe/Athens", "Europe/Athens"), + ("Europe/Belgrade", "Europe/Belgrade"), + ("Europe/Berlin", "Europe/Berlin"), + ("Europe/Brussels", "Europe/Brussels"), + ("Europe/Bucharest", "Europe/Bucharest"), + ("Europe/Budapest", "Europe/Budapest"), + ("Europe/Chisinau", "Europe/Chisinau"), + ("Europe/Copenhagen", "Europe/Copenhagen"), + ("Europe/Dublin", "Europe/Dublin"), + ("Europe/Gibraltar", "Europe/Gibraltar"), + ("Europe/Helsinki", "Europe/Helsinki"), + ("Europe/Istanbul", "Europe/Istanbul"), + ("Europe/Kaliningrad", "Europe/Kaliningrad"), + ("Europe/Kirov", "Europe/Kirov"), + ("Europe/Kyiv", "Europe/Kyiv"), + ("Europe/Lisbon", "Europe/Lisbon"), + ("Europe/London", "Europe/London"), + ("Europe/Luxembourg", "Europe/Luxembourg"), + ("Europe/Madrid", "Europe/Madrid"), + ("Europe/Malta", "Europe/Malta"), + ("Europe/Minsk", "Europe/Minsk"), + ("Europe/Monaco", "Europe/Monaco"), + ("Europe/Moscow", "Europe/Moscow"), + ("Europe/Oslo", "Europe/Oslo"), + ("Europe/Paris", "Europe/Paris"), + ("Europe/Prague", "Europe/Prague"), + ("Europe/Riga", "Europe/Riga"), + ("Europe/Rome", "Europe/Rome"), + ("Europe/Samara", "Europe/Samara"), + ("Europe/Saratov", "Europe/Saratov"), + ("Europe/Simferopol", "Europe/Simferopol"), + ("Europe/Sofia", "Europe/Sofia"), + ("Europe/Stockholm", "Europe/Stockholm"), + ("Europe/Tallinn", "Europe/Tallinn"), + ("Europe/Tirane", "Europe/Tirane"), + ("Europe/Ulyanovsk", "Europe/Ulyanovsk"), + ("Europe/Uzhgorod", "Europe/Uzhgorod"), + ("Europe/Vaduz", "Europe/Vaduz"), + ("Europe/Vienna", "Europe/Vienna"), + ("Europe/Vilnius", "Europe/Vilnius"), + ("Europe/Volgograd", "Europe/Volgograd"), + ("Europe/Warsaw", "Europe/Warsaw"), + ("Europe/Zaporozhye", "Europe/Zaporozhye"), + ("Europe/Zurich", "Europe/Zurich"), + ("Indian/Antananarivo", "Indian/Antananarivo"), + ("Indian/Chagos", "Indian/Chagos"), + ("Indian/Christmas", "Indian/Christmas"), + ("Indian/Cocos", "Indian/Cocos"), + ("Indian/Comoro", "Indian/Comoro"), + ("Indian/Kerguelen", "Indian/Kerguelen"), + ("Indian/Mahe", "Indian/Mahe"), + ("Indian/Maldives", "Indian/Maldives"), + ("Indian/Mauritius", "Indian/Mauritius"), + ("Indian/Mayotte", "Indian/Mayotte"), + ("Indian/Reunion", "Indian/Reunion"), + ("Pacific/Apia", "Pacific/Apia"), + ("Pacific/Auckland", "Pacific/Auckland"), + ("Pacific/Bougainville", "Pacific/Bougainville"), + ("Pacific/Chatham", "Pacific/Chatham"), + ("Pacific/Chuuk", "Pacific/Chuuk"), + ("Pacific/Easter", "Pacific/Easter"), + ("Pacific/Efate", "Pacific/Efate"), + ("Pacific/Fakaofo", "Pacific/Fakaofo"), + ("Pacific/Fiji", "Pacific/Fiji"), + ("Pacific/Funafuti", "Pacific/Funafuti"), + ("Pacific/Galapagos", "Pacific/Galapagos"), + ("Pacific/Gambier", "Pacific/Gambier"), + ("Pacific/Guadalcanal", "Pacific/Guadalcanal"), + ("Pacific/Guam", "Pacific/Guam"), + ("Pacific/Honolulu", "Pacific/Honolulu"), + ("Pacific/Kanton", "Pacific/Kanton"), + ("Pacific/Kiritimati", "Pacific/Kiritimati"), + ("Pacific/Kosrae", "Pacific/Kosrae"), + ("Pacific/Kwajalein", "Pacific/Kwajalein"), + ("Pacific/Majuro", "Pacific/Majuro"), + ("Pacific/Marquesas", "Pacific/Marquesas"), + ("Pacific/Midway", "Pacific/Midway"), + ("Pacific/Nauru", "Pacific/Nauru"), + ("Pacific/Niue", "Pacific/Niue"), + ("Pacific/Norfolk", "Pacific/Norfolk"), + ("Pacific/Noumea", "Pacific/Noumea"), + ("Pacific/Pago_Pago", "Pacific/Pago_Pago"), + ("Pacific/Palau", "Pacific/Palau"), + ("Pacific/Pitcairn", "Pacific/Pitcairn"), + ("Pacific/Pohnpei", "Pacific/Pohnpei"), + ("Pacific/Port_Moresby", "Pacific/Port_Moresby"), + ("Pacific/Rarotonga", "Pacific/Rarotonga"), + ("Pacific/Saipan", "Pacific/Saipan"), + ("Pacific/Tahiti", "Pacific/Tahiti"), + ("Pacific/Tarawa", "Pacific/Tarawa"), + ("Pacific/Tongatapu", "Pacific/Tongatapu"), + ("Pacific/Wake", "Pacific/Wake"), + ("Pacific/Wallis", "Pacific/Wallis"), + ("UTC", "UTC"), + ], + default="UTC", + max_length=255, + ), + ), + ] diff --git a/ietf/meeting/migrations/0016_alter_meeting_country_alter_meeting_time_zone.py b/ietf/meeting/migrations/0016_alter_meeting_country_alter_meeting_time_zone.py new file mode 100644 index 0000000000..8c467ea156 --- /dev/null +++ b/ietf/meeting/migrations/0016_alter_meeting_country_alter_meeting_time_zone.py @@ -0,0 +1,694 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("meeting", "0015_alter_meeting_time_zone"), + ] + + operations = [ + migrations.AlterField( + model_name="meeting", + name="country", + field=models.CharField( + blank=True, + choices=[ + ("", "---------"), + ("AF", "Afghanistan"), + ("AL", "Albania"), + ("DZ", "Algeria"), + ("AD", "Andorra"), + ("AO", "Angola"), + ("AI", "Anguilla"), + ("AQ", "Antarctica"), + ("AG", "Antigua & Barbuda"), + ("AR", "Argentina"), + ("AM", "Armenia"), + ("AW", "Aruba"), + ("AU", "Australia"), + ("AT", "Austria"), + ("AZ", "Azerbaijan"), + ("BS", "Bahamas"), + ("BH", "Bahrain"), + ("BD", "Bangladesh"), + ("BB", "Barbados"), + ("BY", "Belarus"), + ("BE", "Belgium"), + ("BZ", "Belize"), + ("BJ", "Benin"), + ("BM", "Bermuda"), + ("BT", "Bhutan"), + ("BO", "Bolivia"), + ("BA", "Bosnia & Herzegovina"), + ("BW", "Botswana"), + ("BV", "Bouvet Island"), + ("BR", "Brazil"), + ("GB", "Britain (UK)"), + ("IO", "British Indian Ocean Territory"), + ("BN", "Brunei"), + ("BG", "Bulgaria"), + ("BF", "Burkina Faso"), + ("BI", "Burundi"), + ("KH", "Cambodia"), + ("CM", "Cameroon"), + ("CA", "Canada"), + ("CV", "Cape Verde"), + ("BQ", "Caribbean NL"), + ("KY", "Cayman Islands"), + ("CF", "Central African Rep."), + ("TD", "Chad"), + ("CL", "Chile"), + ("CN", "China"), + ("CX", "Christmas Island"), + ("CC", "Cocos (Keeling) Islands"), + ("CO", "Colombia"), + ("KM", "Comoros"), + ("CD", "Congo (Dem. Rep.)"), + ("CG", "Congo (Rep.)"), + ("CK", "Cook Islands"), + ("CR", "Costa Rica"), + ("HR", "Croatia"), + ("CU", "Cuba"), + ("CW", "Curaçao"), + ("CY", "Cyprus"), + ("CZ", "Czech Republic"), + ("CI", "Côte d'Ivoire"), + ("DK", "Denmark"), + ("DJ", "Djibouti"), + ("DM", "Dominica"), + ("DO", "Dominican Republic"), + ("TL", "East Timor"), + ("EC", "Ecuador"), + ("EG", "Egypt"), + ("SV", "El Salvador"), + ("GQ", "Equatorial Guinea"), + ("ER", "Eritrea"), + ("EE", "Estonia"), + ("SZ", "Eswatini (Swaziland)"), + ("ET", "Ethiopia"), + ("FK", "Falkland Islands"), + ("FO", "Faroe Islands"), + ("FJ", "Fiji"), + ("FI", "Finland"), + ("FR", "France"), + ("GF", "French Guiana"), + ("PF", "French Polynesia"), + ("TF", "French S. Terr."), + ("GA", "Gabon"), + ("GM", "Gambia"), + ("GE", "Georgia"), + ("DE", "Germany"), + ("GH", "Ghana"), + ("GI", "Gibraltar"), + ("GR", "Greece"), + ("GL", "Greenland"), + ("GD", "Grenada"), + ("GP", "Guadeloupe"), + ("GU", "Guam"), + ("GT", "Guatemala"), + ("GG", "Guernsey"), + ("GN", "Guinea"), + ("GW", "Guinea-Bissau"), + ("GY", "Guyana"), + ("HT", "Haiti"), + ("HM", "Heard Island & McDonald Islands"), + ("HN", "Honduras"), + ("HK", "Hong Kong"), + ("HU", "Hungary"), + ("IS", "Iceland"), + ("IN", "India"), + ("ID", "Indonesia"), + ("IR", "Iran"), + ("IQ", "Iraq"), + ("IE", "Ireland"), + ("IM", "Isle of Man"), + ("IL", "Israel"), + ("IT", "Italy"), + ("JM", "Jamaica"), + ("JP", "Japan"), + ("JE", "Jersey"), + ("JO", "Jordan"), + ("KZ", "Kazakhstan"), + ("KE", "Kenya"), + ("KI", "Kiribati"), + ("KP", "Korea (North)"), + ("KR", "Korea (South)"), + ("KW", "Kuwait"), + ("KG", "Kyrgyzstan"), + ("LA", "Laos"), + ("LV", "Latvia"), + ("LB", "Lebanon"), + ("LS", "Lesotho"), + ("LR", "Liberia"), + ("LY", "Libya"), + ("LI", "Liechtenstein"), + ("LT", "Lithuania"), + ("LU", "Luxembourg"), + ("MO", "Macau"), + ("MG", "Madagascar"), + ("MW", "Malawi"), + ("MY", "Malaysia"), + ("MV", "Maldives"), + ("ML", "Mali"), + ("MT", "Malta"), + ("MH", "Marshall Islands"), + ("MQ", "Martinique"), + ("MR", "Mauritania"), + ("MU", "Mauritius"), + ("YT", "Mayotte"), + ("MX", "Mexico"), + ("FM", "Micronesia"), + ("MD", "Moldova"), + ("MC", "Monaco"), + ("MN", "Mongolia"), + ("ME", "Montenegro"), + ("MS", "Montserrat"), + ("MA", "Morocco"), + ("MZ", "Mozambique"), + ("MM", "Myanmar (Burma)"), + ("NA", "Namibia"), + ("NR", "Nauru"), + ("NP", "Nepal"), + ("NL", "Netherlands"), + ("NC", "New Caledonia"), + ("NZ", "New Zealand"), + ("NI", "Nicaragua"), + ("NE", "Niger"), + ("NG", "Nigeria"), + ("NU", "Niue"), + ("NF", "Norfolk Island"), + ("MK", "North Macedonia"), + ("MP", "Northern Mariana Islands"), + ("NO", "Norway"), + ("OM", "Oman"), + ("PK", "Pakistan"), + ("PW", "Palau"), + ("PS", "Palestine"), + ("PA", "Panama"), + ("PG", "Papua New Guinea"), + ("PY", "Paraguay"), + ("PE", "Peru"), + ("PH", "Philippines"), + ("PN", "Pitcairn"), + ("PL", "Poland"), + ("PT", "Portugal"), + ("PR", "Puerto Rico"), + ("QA", "Qatar"), + ("RO", "Romania"), + ("RU", "Russia"), + ("RW", "Rwanda"), + ("RE", "Réunion"), + ("AS", "Samoa (American)"), + ("WS", "Samoa (western)"), + ("SM", "San Marino"), + ("ST", "Sao Tome & Principe"), + ("SA", "Saudi Arabia"), + ("SN", "Senegal"), + ("RS", "Serbia"), + ("SC", "Seychelles"), + ("SL", "Sierra Leone"), + ("SG", "Singapore"), + ("SK", "Slovakia"), + ("SI", "Slovenia"), + ("SB", "Solomon Islands"), + ("SO", "Somalia"), + ("ZA", "South Africa"), + ("GS", "South Georgia & the South Sandwich Islands"), + ("SS", "South Sudan"), + ("ES", "Spain"), + ("LK", "Sri Lanka"), + ("BL", "St Barthelemy"), + ("SH", "St Helena"), + ("KN", "St Kitts & Nevis"), + ("LC", "St Lucia"), + ("SX", "St Maarten (Dutch)"), + ("MF", "St Martin (French)"), + ("PM", "St Pierre & Miquelon"), + ("VC", "St Vincent"), + ("SD", "Sudan"), + ("SR", "Suriname"), + ("SJ", "Svalbard & Jan Mayen"), + ("SE", "Sweden"), + ("CH", "Switzerland"), + ("SY", "Syria"), + ("TW", "Taiwan"), + ("TJ", "Tajikistan"), + ("TZ", "Tanzania"), + ("TH", "Thailand"), + ("TG", "Togo"), + ("TK", "Tokelau"), + ("TO", "Tonga"), + ("TT", "Trinidad & Tobago"), + ("TN", "Tunisia"), + ("TR", "Turkey"), + ("TM", "Turkmenistan"), + ("TC", "Turks & Caicos Is"), + ("TV", "Tuvalu"), + ("UM", "US minor outlying islands"), + ("UG", "Uganda"), + ("UA", "Ukraine"), + ("AE", "United Arab Emirates"), + ("US", "United States"), + ("UY", "Uruguay"), + ("UZ", "Uzbekistan"), + ("VU", "Vanuatu"), + ("VA", "Vatican City"), + ("VE", "Venezuela"), + ("VN", "Vietnam"), + ("VG", "Virgin Islands (UK)"), + ("VI", "Virgin Islands (US)"), + ("WF", "Wallis & Futuna"), + ("EH", "Western Sahara"), + ("YE", "Yemen"), + ("ZM", "Zambia"), + ("ZW", "Zimbabwe"), + ("AX", "Åland Islands"), + ], + max_length=2, + ), + ), + migrations.AlterField( + model_name="meeting", + name="time_zone", + field=models.CharField( + choices=[ + ("", "---------"), + ("Africa/Abidjan", "Africa/Abidjan"), + ("Africa/Accra", "Africa/Accra"), + ("Africa/Addis_Ababa", "Africa/Addis_Ababa"), + ("Africa/Algiers", "Africa/Algiers"), + ("Africa/Asmara", "Africa/Asmara"), + ("Africa/Bamako", "Africa/Bamako"), + ("Africa/Bangui", "Africa/Bangui"), + ("Africa/Banjul", "Africa/Banjul"), + ("Africa/Bissau", "Africa/Bissau"), + ("Africa/Blantyre", "Africa/Blantyre"), + ("Africa/Brazzaville", "Africa/Brazzaville"), + ("Africa/Bujumbura", "Africa/Bujumbura"), + ("Africa/Cairo", "Africa/Cairo"), + ("Africa/Casablanca", "Africa/Casablanca"), + ("Africa/Ceuta", "Africa/Ceuta"), + ("Africa/Conakry", "Africa/Conakry"), + ("Africa/Dakar", "Africa/Dakar"), + ("Africa/Dar_es_Salaam", "Africa/Dar_es_Salaam"), + ("Africa/Djibouti", "Africa/Djibouti"), + ("Africa/Douala", "Africa/Douala"), + ("Africa/El_Aaiun", "Africa/El_Aaiun"), + ("Africa/Freetown", "Africa/Freetown"), + ("Africa/Gaborone", "Africa/Gaborone"), + ("Africa/Harare", "Africa/Harare"), + ("Africa/Johannesburg", "Africa/Johannesburg"), + ("Africa/Juba", "Africa/Juba"), + ("Africa/Kampala", "Africa/Kampala"), + ("Africa/Khartoum", "Africa/Khartoum"), + ("Africa/Kigali", "Africa/Kigali"), + ("Africa/Kinshasa", "Africa/Kinshasa"), + ("Africa/Lagos", "Africa/Lagos"), + ("Africa/Libreville", "Africa/Libreville"), + ("Africa/Lome", "Africa/Lome"), + ("Africa/Luanda", "Africa/Luanda"), + ("Africa/Lubumbashi", "Africa/Lubumbashi"), + ("Africa/Lusaka", "Africa/Lusaka"), + ("Africa/Malabo", "Africa/Malabo"), + ("Africa/Maputo", "Africa/Maputo"), + ("Africa/Maseru", "Africa/Maseru"), + ("Africa/Mbabane", "Africa/Mbabane"), + ("Africa/Mogadishu", "Africa/Mogadishu"), + ("Africa/Monrovia", "Africa/Monrovia"), + ("Africa/Nairobi", "Africa/Nairobi"), + ("Africa/Ndjamena", "Africa/Ndjamena"), + ("Africa/Niamey", "Africa/Niamey"), + ("Africa/Nouakchott", "Africa/Nouakchott"), + ("Africa/Ouagadougou", "Africa/Ouagadougou"), + ("Africa/Porto-Novo", "Africa/Porto-Novo"), + ("Africa/Sao_Tome", "Africa/Sao_Tome"), + ("Africa/Tripoli", "Africa/Tripoli"), + ("Africa/Tunis", "Africa/Tunis"), + ("Africa/Windhoek", "Africa/Windhoek"), + ("America/Adak", "America/Adak"), + ("America/Anchorage", "America/Anchorage"), + ("America/Anguilla", "America/Anguilla"), + ("America/Antigua", "America/Antigua"), + ("America/Araguaina", "America/Araguaina"), + ( + "America/Argentina/Buenos_Aires", + "America/Argentina/Buenos_Aires", + ), + ("America/Argentina/Catamarca", "America/Argentina/Catamarca"), + ("America/Argentina/Cordoba", "America/Argentina/Cordoba"), + ("America/Argentina/Jujuy", "America/Argentina/Jujuy"), + ("America/Argentina/La_Rioja", "America/Argentina/La_Rioja"), + ("America/Argentina/Mendoza", "America/Argentina/Mendoza"), + ( + "America/Argentina/Rio_Gallegos", + "America/Argentina/Rio_Gallegos", + ), + ("America/Argentina/Salta", "America/Argentina/Salta"), + ("America/Argentina/San_Juan", "America/Argentina/San_Juan"), + ("America/Argentina/San_Luis", "America/Argentina/San_Luis"), + ("America/Argentina/Tucuman", "America/Argentina/Tucuman"), + ("America/Argentina/Ushuaia", "America/Argentina/Ushuaia"), + ("America/Aruba", "America/Aruba"), + ("America/Asuncion", "America/Asuncion"), + ("America/Atikokan", "America/Atikokan"), + ("America/Bahia", "America/Bahia"), + ("America/Bahia_Banderas", "America/Bahia_Banderas"), + ("America/Barbados", "America/Barbados"), + ("America/Belem", "America/Belem"), + ("America/Belize", "America/Belize"), + ("America/Blanc-Sablon", "America/Blanc-Sablon"), + ("America/Boa_Vista", "America/Boa_Vista"), + ("America/Bogota", "America/Bogota"), + ("America/Boise", "America/Boise"), + ("America/Cambridge_Bay", "America/Cambridge_Bay"), + ("America/Campo_Grande", "America/Campo_Grande"), + ("America/Cancun", "America/Cancun"), + ("America/Caracas", "America/Caracas"), + ("America/Cayenne", "America/Cayenne"), + ("America/Cayman", "America/Cayman"), + ("America/Chicago", "America/Chicago"), + ("America/Chihuahua", "America/Chihuahua"), + ("America/Ciudad_Juarez", "America/Ciudad_Juarez"), + ("America/Costa_Rica", "America/Costa_Rica"), + ("America/Coyhaique", "America/Coyhaique"), + ("America/Creston", "America/Creston"), + ("America/Cuiaba", "America/Cuiaba"), + ("America/Curacao", "America/Curacao"), + ("America/Danmarkshavn", "America/Danmarkshavn"), + ("America/Dawson", "America/Dawson"), + ("America/Dawson_Creek", "America/Dawson_Creek"), + ("America/Denver", "America/Denver"), + ("America/Detroit", "America/Detroit"), + ("America/Dominica", "America/Dominica"), + ("America/Edmonton", "America/Edmonton"), + ("America/Eirunepe", "America/Eirunepe"), + ("America/El_Salvador", "America/El_Salvador"), + ("America/Fort_Nelson", "America/Fort_Nelson"), + ("America/Fortaleza", "America/Fortaleza"), + ("America/Glace_Bay", "America/Glace_Bay"), + ("America/Goose_Bay", "America/Goose_Bay"), + ("America/Grand_Turk", "America/Grand_Turk"), + ("America/Grenada", "America/Grenada"), + ("America/Guadeloupe", "America/Guadeloupe"), + ("America/Guatemala", "America/Guatemala"), + ("America/Guayaquil", "America/Guayaquil"), + ("America/Guyana", "America/Guyana"), + ("America/Halifax", "America/Halifax"), + ("America/Havana", "America/Havana"), + ("America/Hermosillo", "America/Hermosillo"), + ("America/Indiana/Indianapolis", "America/Indiana/Indianapolis"), + ("America/Indiana/Knox", "America/Indiana/Knox"), + ("America/Indiana/Marengo", "America/Indiana/Marengo"), + ("America/Indiana/Petersburg", "America/Indiana/Petersburg"), + ("America/Indiana/Tell_City", "America/Indiana/Tell_City"), + ("America/Indiana/Vevay", "America/Indiana/Vevay"), + ("America/Indiana/Vincennes", "America/Indiana/Vincennes"), + ("America/Indiana/Winamac", "America/Indiana/Winamac"), + ("America/Inuvik", "America/Inuvik"), + ("America/Iqaluit", "America/Iqaluit"), + ("America/Jamaica", "America/Jamaica"), + ("America/Juneau", "America/Juneau"), + ("America/Kentucky/Louisville", "America/Kentucky/Louisville"), + ("America/Kentucky/Monticello", "America/Kentucky/Monticello"), + ("America/La_Paz", "America/La_Paz"), + ("America/Lima", "America/Lima"), + ("America/Los_Angeles", "America/Los_Angeles"), + ("America/Maceio", "America/Maceio"), + ("America/Managua", "America/Managua"), + ("America/Manaus", "America/Manaus"), + ("America/Martinique", "America/Martinique"), + ("America/Matamoros", "America/Matamoros"), + ("America/Mazatlan", "America/Mazatlan"), + ("America/Menominee", "America/Menominee"), + ("America/Merida", "America/Merida"), + ("America/Metlakatla", "America/Metlakatla"), + ("America/Mexico_City", "America/Mexico_City"), + ("America/Miquelon", "America/Miquelon"), + ("America/Moncton", "America/Moncton"), + ("America/Monterrey", "America/Monterrey"), + ("America/Montevideo", "America/Montevideo"), + ("America/Montserrat", "America/Montserrat"), + ("America/Nassau", "America/Nassau"), + ("America/New_York", "America/New_York"), + ("America/Nome", "America/Nome"), + ("America/Noronha", "America/Noronha"), + ("America/North_Dakota/Beulah", "America/North_Dakota/Beulah"), + ("America/North_Dakota/Center", "America/North_Dakota/Center"), + ( + "America/North_Dakota/New_Salem", + "America/North_Dakota/New_Salem", + ), + ("America/Nuuk", "America/Nuuk"), + ("America/Ojinaga", "America/Ojinaga"), + ("America/Panama", "America/Panama"), + ("America/Paramaribo", "America/Paramaribo"), + ("America/Phoenix", "America/Phoenix"), + ("America/Port-au-Prince", "America/Port-au-Prince"), + ("America/Port_of_Spain", "America/Port_of_Spain"), + ("America/Porto_Velho", "America/Porto_Velho"), + ("America/Puerto_Rico", "America/Puerto_Rico"), + ("America/Punta_Arenas", "America/Punta_Arenas"), + ("America/Rankin_Inlet", "America/Rankin_Inlet"), + ("America/Recife", "America/Recife"), + ("America/Regina", "America/Regina"), + ("America/Resolute", "America/Resolute"), + ("America/Rio_Branco", "America/Rio_Branco"), + ("America/Santarem", "America/Santarem"), + ("America/Santiago", "America/Santiago"), + ("America/Santo_Domingo", "America/Santo_Domingo"), + ("America/Sao_Paulo", "America/Sao_Paulo"), + ("America/Scoresbysund", "America/Scoresbysund"), + ("America/Sitka", "America/Sitka"), + ("America/St_Johns", "America/St_Johns"), + ("America/St_Kitts", "America/St_Kitts"), + ("America/St_Lucia", "America/St_Lucia"), + ("America/St_Thomas", "America/St_Thomas"), + ("America/St_Vincent", "America/St_Vincent"), + ("America/Swift_Current", "America/Swift_Current"), + ("America/Tegucigalpa", "America/Tegucigalpa"), + ("America/Thule", "America/Thule"), + ("America/Tijuana", "America/Tijuana"), + ("America/Toronto", "America/Toronto"), + ("America/Tortola", "America/Tortola"), + ("America/Vancouver", "America/Vancouver"), + ("America/Whitehorse", "America/Whitehorse"), + ("America/Winnipeg", "America/Winnipeg"), + ("America/Yakutat", "America/Yakutat"), + ("Antarctica/Casey", "Antarctica/Casey"), + ("Antarctica/Davis", "Antarctica/Davis"), + ("Antarctica/DumontDUrville", "Antarctica/DumontDUrville"), + ("Antarctica/Macquarie", "Antarctica/Macquarie"), + ("Antarctica/Mawson", "Antarctica/Mawson"), + ("Antarctica/McMurdo", "Antarctica/McMurdo"), + ("Antarctica/Palmer", "Antarctica/Palmer"), + ("Antarctica/Rothera", "Antarctica/Rothera"), + ("Antarctica/Syowa", "Antarctica/Syowa"), + ("Antarctica/Troll", "Antarctica/Troll"), + ("Antarctica/Vostok", "Antarctica/Vostok"), + ("Asia/Aden", "Asia/Aden"), + ("Asia/Almaty", "Asia/Almaty"), + ("Asia/Amman", "Asia/Amman"), + ("Asia/Anadyr", "Asia/Anadyr"), + ("Asia/Aqtau", "Asia/Aqtau"), + ("Asia/Aqtobe", "Asia/Aqtobe"), + ("Asia/Ashgabat", "Asia/Ashgabat"), + ("Asia/Atyrau", "Asia/Atyrau"), + ("Asia/Baghdad", "Asia/Baghdad"), + ("Asia/Bahrain", "Asia/Bahrain"), + ("Asia/Baku", "Asia/Baku"), + ("Asia/Bangkok", "Asia/Bangkok"), + ("Asia/Barnaul", "Asia/Barnaul"), + ("Asia/Beirut", "Asia/Beirut"), + ("Asia/Bishkek", "Asia/Bishkek"), + ("Asia/Brunei", "Asia/Brunei"), + ("Asia/Chita", "Asia/Chita"), + ("Asia/Colombo", "Asia/Colombo"), + ("Asia/Damascus", "Asia/Damascus"), + ("Asia/Dhaka", "Asia/Dhaka"), + ("Asia/Dili", "Asia/Dili"), + ("Asia/Dubai", "Asia/Dubai"), + ("Asia/Dushanbe", "Asia/Dushanbe"), + ("Asia/Famagusta", "Asia/Famagusta"), + ("Asia/Gaza", "Asia/Gaza"), + ("Asia/Hebron", "Asia/Hebron"), + ("Asia/Ho_Chi_Minh", "Asia/Ho_Chi_Minh"), + ("Asia/Hong_Kong", "Asia/Hong_Kong"), + ("Asia/Hovd", "Asia/Hovd"), + ("Asia/Irkutsk", "Asia/Irkutsk"), + ("Asia/Jakarta", "Asia/Jakarta"), + ("Asia/Jayapura", "Asia/Jayapura"), + ("Asia/Jerusalem", "Asia/Jerusalem"), + ("Asia/Kabul", "Asia/Kabul"), + ("Asia/Kamchatka", "Asia/Kamchatka"), + ("Asia/Karachi", "Asia/Karachi"), + ("Asia/Kathmandu", "Asia/Kathmandu"), + ("Asia/Khandyga", "Asia/Khandyga"), + ("Asia/Kolkata", "Asia/Kolkata"), + ("Asia/Krasnoyarsk", "Asia/Krasnoyarsk"), + ("Asia/Kuala_Lumpur", "Asia/Kuala_Lumpur"), + ("Asia/Kuching", "Asia/Kuching"), + ("Asia/Kuwait", "Asia/Kuwait"), + ("Asia/Macau", "Asia/Macau"), + ("Asia/Magadan", "Asia/Magadan"), + ("Asia/Makassar", "Asia/Makassar"), + ("Asia/Manila", "Asia/Manila"), + ("Asia/Muscat", "Asia/Muscat"), + ("Asia/Nicosia", "Asia/Nicosia"), + ("Asia/Novokuznetsk", "Asia/Novokuznetsk"), + ("Asia/Novosibirsk", "Asia/Novosibirsk"), + ("Asia/Omsk", "Asia/Omsk"), + ("Asia/Oral", "Asia/Oral"), + ("Asia/Phnom_Penh", "Asia/Phnom_Penh"), + ("Asia/Pontianak", "Asia/Pontianak"), + ("Asia/Pyongyang", "Asia/Pyongyang"), + ("Asia/Qatar", "Asia/Qatar"), + ("Asia/Qostanay", "Asia/Qostanay"), + ("Asia/Qyzylorda", "Asia/Qyzylorda"), + ("Asia/Riyadh", "Asia/Riyadh"), + ("Asia/Sakhalin", "Asia/Sakhalin"), + ("Asia/Samarkand", "Asia/Samarkand"), + ("Asia/Seoul", "Asia/Seoul"), + ("Asia/Shanghai", "Asia/Shanghai"), + ("Asia/Singapore", "Asia/Singapore"), + ("Asia/Srednekolymsk", "Asia/Srednekolymsk"), + ("Asia/Taipei", "Asia/Taipei"), + ("Asia/Tashkent", "Asia/Tashkent"), + ("Asia/Tbilisi", "Asia/Tbilisi"), + ("Asia/Tehran", "Asia/Tehran"), + ("Asia/Thimphu", "Asia/Thimphu"), + ("Asia/Tokyo", "Asia/Tokyo"), + ("Asia/Tomsk", "Asia/Tomsk"), + ("Asia/Ulaanbaatar", "Asia/Ulaanbaatar"), + ("Asia/Urumqi", "Asia/Urumqi"), + ("Asia/Ust-Nera", "Asia/Ust-Nera"), + ("Asia/Vientiane", "Asia/Vientiane"), + ("Asia/Vladivostok", "Asia/Vladivostok"), + ("Asia/Yakutsk", "Asia/Yakutsk"), + ("Asia/Yangon", "Asia/Yangon"), + ("Asia/Yekaterinburg", "Asia/Yekaterinburg"), + ("Asia/Yerevan", "Asia/Yerevan"), + ("Atlantic/Azores", "Atlantic/Azores"), + ("Atlantic/Bermuda", "Atlantic/Bermuda"), + ("Atlantic/Canary", "Atlantic/Canary"), + ("Atlantic/Cape_Verde", "Atlantic/Cape_Verde"), + ("Atlantic/Faroe", "Atlantic/Faroe"), + ("Atlantic/Madeira", "Atlantic/Madeira"), + ("Atlantic/Reykjavik", "Atlantic/Reykjavik"), + ("Atlantic/South_Georgia", "Atlantic/South_Georgia"), + ("Atlantic/St_Helena", "Atlantic/St_Helena"), + ("Atlantic/Stanley", "Atlantic/Stanley"), + ("Australia/Adelaide", "Australia/Adelaide"), + ("Australia/Brisbane", "Australia/Brisbane"), + ("Australia/Broken_Hill", "Australia/Broken_Hill"), + ("Australia/Darwin", "Australia/Darwin"), + ("Australia/Eucla", "Australia/Eucla"), + ("Australia/Hobart", "Australia/Hobart"), + ("Australia/Lindeman", "Australia/Lindeman"), + ("Australia/Lord_Howe", "Australia/Lord_Howe"), + ("Australia/Melbourne", "Australia/Melbourne"), + ("Australia/Perth", "Australia/Perth"), + ("Australia/Sydney", "Australia/Sydney"), + ("Europe/Amsterdam", "Europe/Amsterdam"), + ("Europe/Andorra", "Europe/Andorra"), + ("Europe/Astrakhan", "Europe/Astrakhan"), + ("Europe/Athens", "Europe/Athens"), + ("Europe/Belgrade", "Europe/Belgrade"), + ("Europe/Berlin", "Europe/Berlin"), + ("Europe/Brussels", "Europe/Brussels"), + ("Europe/Bucharest", "Europe/Bucharest"), + ("Europe/Budapest", "Europe/Budapest"), + ("Europe/Chisinau", "Europe/Chisinau"), + ("Europe/Copenhagen", "Europe/Copenhagen"), + ("Europe/Dublin", "Europe/Dublin"), + ("Europe/Gibraltar", "Europe/Gibraltar"), + ("Europe/Helsinki", "Europe/Helsinki"), + ("Europe/Istanbul", "Europe/Istanbul"), + ("Europe/Kaliningrad", "Europe/Kaliningrad"), + ("Europe/Kirov", "Europe/Kirov"), + ("Europe/Kyiv", "Europe/Kyiv"), + ("Europe/Lisbon", "Europe/Lisbon"), + ("Europe/London", "Europe/London"), + ("Europe/Luxembourg", "Europe/Luxembourg"), + ("Europe/Madrid", "Europe/Madrid"), + ("Europe/Malta", "Europe/Malta"), + ("Europe/Minsk", "Europe/Minsk"), + ("Europe/Monaco", "Europe/Monaco"), + ("Europe/Moscow", "Europe/Moscow"), + ("Europe/Oslo", "Europe/Oslo"), + ("Europe/Paris", "Europe/Paris"), + ("Europe/Prague", "Europe/Prague"), + ("Europe/Riga", "Europe/Riga"), + ("Europe/Rome", "Europe/Rome"), + ("Europe/Samara", "Europe/Samara"), + ("Europe/Saratov", "Europe/Saratov"), + ("Europe/Simferopol", "Europe/Simferopol"), + ("Europe/Sofia", "Europe/Sofia"), + ("Europe/Stockholm", "Europe/Stockholm"), + ("Europe/Tallinn", "Europe/Tallinn"), + ("Europe/Tirane", "Europe/Tirane"), + ("Europe/Ulyanovsk", "Europe/Ulyanovsk"), + ("Europe/Vaduz", "Europe/Vaduz"), + ("Europe/Vienna", "Europe/Vienna"), + ("Europe/Vilnius", "Europe/Vilnius"), + ("Europe/Volgograd", "Europe/Volgograd"), + ("Europe/Warsaw", "Europe/Warsaw"), + ("Europe/Zurich", "Europe/Zurich"), + ("Indian/Antananarivo", "Indian/Antananarivo"), + ("Indian/Chagos", "Indian/Chagos"), + ("Indian/Christmas", "Indian/Christmas"), + ("Indian/Cocos", "Indian/Cocos"), + ("Indian/Comoro", "Indian/Comoro"), + ("Indian/Kerguelen", "Indian/Kerguelen"), + ("Indian/Mahe", "Indian/Mahe"), + ("Indian/Maldives", "Indian/Maldives"), + ("Indian/Mauritius", "Indian/Mauritius"), + ("Indian/Mayotte", "Indian/Mayotte"), + ("Indian/Reunion", "Indian/Reunion"), + ("Pacific/Apia", "Pacific/Apia"), + ("Pacific/Auckland", "Pacific/Auckland"), + ("Pacific/Bougainville", "Pacific/Bougainville"), + ("Pacific/Chatham", "Pacific/Chatham"), + ("Pacific/Chuuk", "Pacific/Chuuk"), + ("Pacific/Easter", "Pacific/Easter"), + ("Pacific/Efate", "Pacific/Efate"), + ("Pacific/Fakaofo", "Pacific/Fakaofo"), + ("Pacific/Fiji", "Pacific/Fiji"), + ("Pacific/Funafuti", "Pacific/Funafuti"), + ("Pacific/Galapagos", "Pacific/Galapagos"), + ("Pacific/Gambier", "Pacific/Gambier"), + ("Pacific/Guadalcanal", "Pacific/Guadalcanal"), + ("Pacific/Guam", "Pacific/Guam"), + ("Pacific/Honolulu", "Pacific/Honolulu"), + ("Pacific/Kanton", "Pacific/Kanton"), + ("Pacific/Kiritimati", "Pacific/Kiritimati"), + ("Pacific/Kosrae", "Pacific/Kosrae"), + ("Pacific/Kwajalein", "Pacific/Kwajalein"), + ("Pacific/Majuro", "Pacific/Majuro"), + ("Pacific/Marquesas", "Pacific/Marquesas"), + ("Pacific/Midway", "Pacific/Midway"), + ("Pacific/Nauru", "Pacific/Nauru"), + ("Pacific/Niue", "Pacific/Niue"), + ("Pacific/Norfolk", "Pacific/Norfolk"), + ("Pacific/Noumea", "Pacific/Noumea"), + ("Pacific/Pago_Pago", "Pacific/Pago_Pago"), + ("Pacific/Palau", "Pacific/Palau"), + ("Pacific/Pitcairn", "Pacific/Pitcairn"), + ("Pacific/Pohnpei", "Pacific/Pohnpei"), + ("Pacific/Port_Moresby", "Pacific/Port_Moresby"), + ("Pacific/Rarotonga", "Pacific/Rarotonga"), + ("Pacific/Saipan", "Pacific/Saipan"), + ("Pacific/Tahiti", "Pacific/Tahiti"), + ("Pacific/Tarawa", "Pacific/Tarawa"), + ("Pacific/Tongatapu", "Pacific/Tongatapu"), + ("Pacific/Wake", "Pacific/Wake"), + ("Pacific/Wallis", "Pacific/Wallis"), + ("UTC", "UTC"), + ], + default="UTC", + max_length=255, + ), + ), + ] diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py index 84d151c310..7d9e318aab 100644 --- a/ietf/meeting/models.py +++ b/ietf/meeting/models.py @@ -1,5 +1,4 @@ -# Copyright The IETF Trust 2007-2024, All Rights Reserved -# -*- coding: utf-8 -*- +# Copyright The IETF Trust 2007-2025, All Rights Reserved # old meeting models can be found in ../proceedings/models.py @@ -34,7 +33,7 @@ from ietf.name.models import ( MeetingTypeName, TimeSlotTypeName, SessionStatusName, ConstraintName, RoomResourceName, ImportantDateName, TimerangeName, SlideSubmissionStatusName, ProceedingsMaterialTypeName, - SessionPurposeName, + SessionPurposeName, AttendanceTypeName, RegistrationTicketTypeName ) from ietf.person.models import Person from ietf.utils.decorators import memoize @@ -49,15 +48,20 @@ ) from ietf.utils.fields import MissingOkImageField -countries = list(pytz.country_names.items()) -countries.sort(key=lambda x: x[1]) +# Set up countries / timezones, including an empty choice for fields +EMPTY_CHOICE = ("", "-" * 9) +COUNTRIES = (EMPTY_CHOICE,) + tuple( + sorted(pytz.country_names.items(), key=lambda x: x[1]) +) -timezones = [] -for name in pytz.common_timezones: - tzfn = os.path.join(settings.TZDATA_ICS_PATH, name + ".ics") - if not os.path.islink(tzfn): - timezones.append((name, name)) -timezones.sort() +_tzdata_ics_path = Path(settings.TZDATA_ICS_PATH) +TIMEZONES = (EMPTY_CHOICE,) + tuple( + sorted( + (name, name) + for name in pytz.common_timezones + if name != "GMT" and not (_tzdata_ics_path / f"{name}.ics").is_symlink() + ) +) class Meeting(models.Model): @@ -72,11 +76,11 @@ class Meeting(models.Model): days = models.IntegerField(default=7, null=False, validators=[MinValueValidator(1)], help_text="The number of days the meeting lasts") city = models.CharField(blank=True, max_length=255) - country = models.CharField(blank=True, max_length=2, choices=countries) + country = models.CharField(blank=True, max_length=2, choices=COUNTRIES) # We can't derive time-zone from country, as there are some that have # more than one timezone, and the pytz module doesn't provide timezone # lookup information for all relevant city/country combinations. - time_zone = models.CharField(max_length=255, choices=timezones, default='UTC') + time_zone = models.CharField(max_length=255, choices=TIMEZONES, default='UTC') idsubmit_cutoff_day_offset_00 = models.IntegerField(blank=True, default=settings.IDSUBMIT_DEFAULT_CUTOFF_DAY_OFFSET_00, help_text = "The number of days before the meeting start date when the submission of -00 drafts will be closed.") @@ -145,7 +149,7 @@ def get_00_cutoff(self): cutoff_date = importantdate.date else: cutoff_date = self.date + datetime.timedelta(days=ImportantDateName.objects.get(slug='idcutoff').default_offset_days) - cutoff_time = datetime_from_date(cutoff_date, datetime.timezone.utc) + self.idsubmit_cutoff_time_utc + cutoff_time = datetime_from_date(cutoff_date, datetime.UTC) + self.idsubmit_cutoff_time_utc return cutoff_time def get_01_cutoff(self): @@ -157,7 +161,7 @@ def get_01_cutoff(self): cutoff_date = importantdate.date else: cutoff_date = self.date + datetime.timedelta(days=ImportantDateName.objects.get(slug='idcutoff').default_offset_days) - cutoff_time = datetime_from_date(cutoff_date, datetime.timezone.utc) + self.idsubmit_cutoff_time_utc + cutoff_time = datetime_from_date(cutoff_date, datetime.UTC) + self.idsubmit_cutoff_time_utc return cutoff_time def get_reopen_time(self): @@ -233,9 +237,9 @@ def get_proceedings_materials(self): ).order_by('type__order') def get_attendance(self): - """Get the meeting attendance from the MeetingRegistrations + """Get the meeting attendance from the Registrations - Returns a NamedTuple with onsite and online attributes. Returns None if the record is unavailable + Returns a NamedTuple with onsite and remote attributes. Returns None if the record is unavailable for this meeting. """ number = self.get_number() @@ -246,25 +250,39 @@ def get_attendance(self): # MeetingRegistration.attended started conflating badge-pickup and session attendance before IETF 114. # We've separated session attendance off to ietf.meeting.Attended, but need to report attendance at older # meetings correctly. - - attended_per_meetingregistration = ( - Q(meetingregistration__meeting=self) & ( - Q(meetingregistration__attended=True) | - Q(meetingregistration__checkedin=True) + # + # Looking up by registration and attendance records separately and joining in + # python is far faster than combining the Q objects in the query (~100x). + # Further optimization may be possible, but the queries are tricky... + attended_per_meeting_registration = ( + Q(registration__meeting=self) & ( + Q(registration__attended=True) | + Q(registration__checkedin=True) ) ) + attendees_by_reg = set( + Person.objects.filter(attended_per_meeting_registration).values_list( + "pk", flat=True + ) + ) + attended_per_meeting_attended = ( Q(attended__session__meeting=self) # Note that we are not filtering to plenary, wg, or rg sessions # as we do for nomcom eligibility - if picking up a badge (see above) # is good enough, just attending e.g. a training session is also good enough ) - attended = Person.objects.filter( - attended_per_meetingregistration | attended_per_meeting_attended - ).distinct() - - onsite=set(attended.filter(meetingregistration__meeting=self, meetingregistration__reg_type='onsite')) - remote=set(attended.filter(meetingregistration__meeting=self, meetingregistration__reg_type='remote')) + attendees_by_att = set( + Person.objects.filter(attended_per_meeting_attended).values_list( + "pk", flat=True + ) + ) + + attendees = Person.objects.filter( + pk__in=attendees_by_att | attendees_by_reg + ) + onsite = set(attendees.filter(registration__meeting=self, registration__tickets__attendance_type__slug='onsite')) + remote = set(attendees.filter(registration__meeting=self, registration__tickets__attendance_type__slug='remote')) remote.difference_update(onsite) return Attendance( @@ -530,7 +548,7 @@ class FloorPlan(models.Model): image = models.ImageField( storage=BlobShadowFileSystemStorage(kind="floorplan"), upload_to=floorplan_path, - blank=True, + blank=False, default=None, ) # @@ -938,8 +956,6 @@ class Meta: def __str__(self): return u"%s -> %s-%s" % (self.session, self.document.name, self.rev) -constraint_cache_uses = 0 -constraint_cache_initials = 0 class SessionQuerySet(models.QuerySet): def with_current_status(self): @@ -1168,7 +1184,7 @@ def can_manage_materials(self, user): return can_manage_materials(user,self.group) def is_material_submission_cutoff(self): - return date_today(datetime.timezone.utc) > self.meeting.get_submission_correction_date() + return date_today(datetime.UTC) > self.meeting.get_submission_correction_date() def joint_with_groups_acronyms(self): return [group.acronym for group in self.joint_with_groups.all()] @@ -1483,3 +1499,48 @@ class Meta: def __str__(self): return f'{self.person} at {self.session}' + + +class RegistrationManager(models.Manager): + def onsite(self): + return self.get_queryset().filter(tickets__attendance_type__slug='onsite') + + def remote(self): + return self.get_queryset().filter(tickets__attendance_type__slug='remote').exclude(tickets__attendance_type__slug='onsite') + +class Registration(models.Model): + """Registration attendee records from the IETF registration system""" + meeting = ForeignKey(Meeting) + first_name = models.CharField(max_length=255) + last_name = models.CharField(max_length=255) + affiliation = models.CharField(blank=True, max_length=255) + country_code = models.CharField(max_length=2) # ISO 3166 + person = ForeignKey(Person, blank=True, null=True, on_delete=models.PROTECT) + email = models.EmailField(blank=True, null=True) + # attended was used prior to the introduction of the ietf.meeting.Attended model and is still used by + # Meeting.get_attendance() for older meetings. It should not be used except for dealing with legacy data. + attended = models.BooleanField(default=False) + # checkedin indicates that the badge was picked up + checkedin = models.BooleanField(default=False) + + # custom manager + objects = RegistrationManager() + + def __str__(self): + return "{} {}".format(self.first_name, self.last_name) + + @property + def attendance_type(self): + if self.tickets.filter(attendance_type__slug='onsite').exists(): + return 'onsite' + elif self.tickets.filter(attendance_type__slug='remote').exists(): + return 'remote' + return None + +class RegistrationTicket(models.Model): + registration = ForeignKey(Registration, related_name='tickets') + attendance_type = ForeignKey(AttendanceTypeName, on_delete=models.PROTECT) + ticket_type = ForeignKey(RegistrationTicketTypeName, on_delete=models.PROTECT) + + def __str__(self): + return "{}:{}".format(self.attendance_type, self.ticket_type) diff --git a/ietf/meeting/resources.py b/ietf/meeting/resources.py index de9ca01476..490b75f925 100644 --- a/ietf/meeting/resources.py +++ b/ietf/meeting/resources.py @@ -11,12 +11,23 @@ from ietf import api -from ietf.meeting.models import ( Meeting, ResourceAssociation, Constraint, Room, Schedule, Session, - TimeSlot, SchedTimeSessAssignment, SessionPresentation, FloorPlan, - UrlResource, ImportantDate, SlideSubmission, SchedulingEvent, - BusinessConstraint, ProceedingsMaterial, MeetingHost, Attended) +from ietf.meeting.models import (Meeting, ResourceAssociation, Constraint, Room, + Schedule, Session, + TimeSlot, SchedTimeSessAssignment, SessionPresentation, + FloorPlan, + UrlResource, ImportantDate, SlideSubmission, + SchedulingEvent, + BusinessConstraint, ProceedingsMaterial, MeetingHost, + Attended, + Registration, RegistrationTicket) + +from ietf.name.resources import ( + AttendanceTypeNameResource, + MeetingTypeNameResource, + RegistrationTicketTypeNameResource, +) + -from ietf.name.resources import MeetingTypeNameResource class MeetingResource(ModelResource): type = ToOneField(MeetingTypeNameResource, 'type') schedule = ToOneField('ietf.meeting.resources.ScheduleResource', 'schedule', null=True) @@ -431,3 +442,52 @@ class Meta: "session": ALL_WITH_RELATIONS, } api.meeting.register(AttendedResource()) + +from ietf.person.resources import PersonResource +class RegistrationResource(ModelResource): + meeting = ToOneField(MeetingResource, 'meeting') + person = ToOneField(PersonResource, 'person', null=True) + tickets = ToManyField( + 'ietf.meeting.resources.RegistrationTicketResource', + 'tickets', + full=True, + ) + + class Meta: + queryset = Registration.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'registration' + ordering = ['id', ] + filtering = { + "id": ALL, + "first_name": ALL, + "last_name": ALL, + "affiliation": ALL, + "country_code": ALL, + "email": ALL, + "attended": ALL, + "checkedin": ALL, + "meeting": ALL_WITH_RELATIONS, + "person": ALL_WITH_RELATIONS, + "tickets": ALL_WITH_RELATIONS, + } +api.meeting.register(RegistrationResource()) + +class RegistrationTicketResource(ModelResource): + registration = ToOneField(RegistrationResource, 'registration') + attendance_type = ToOneField(AttendanceTypeNameResource, 'attendance_type') + ticket_type = ToOneField(RegistrationTicketTypeNameResource, 'ticket_type') + class Meta: + queryset = RegistrationTicket.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'registrationticket' + ordering = ['id', ] + filtering = { + "id": ALL, + "ticket_type": ALL_WITH_RELATIONS, + "attendance_type": ALL_WITH_RELATIONS, + "registration": ALL_WITH_RELATIONS, + } +api.meeting.register(RegistrationTicketResource()) diff --git a/ietf/meeting/tasks.py b/ietf/meeting/tasks.py index 2b7c2fca91..a73763560b 100644 --- a/ietf/meeting/tasks.py +++ b/ietf/meeting/tasks.py @@ -1,19 +1,77 @@ -# Copyright The IETF Trust 2024, All Rights Reserved +# Copyright The IETF Trust 2024-2026, All Rights Reserved # # Celery task definitions # -from celery import shared_task +import datetime + +from itertools import batched + +from celery import shared_task, chain +from django.db.models import IntegerField +from django.db.models.functions import Cast from django.utils import timezone from ietf.utils import log from .models import Meeting -from .utils import generate_proceedings_content +from .utils import ( + generate_proceedings_content, + resolve_materials_for_one_meeting, + store_blobs_for_one_meeting, +) from .views import generate_agenda_data +from .utils import fetch_attendance_from_meetings + + +@shared_task +def agenda_data_refresh_task(num=None): + """Refresh agenda data for one plenary meeting + + If `num` is `None`, refreshes data for the current meeting. + """ + log.log( + f"Refreshing agenda data for {f"IETF-{num}" if num else "current IETF meeting"}" + ) + try: + generate_agenda_data(num, force_refresh=True) + except Exception as err: + # Log and swallow exceptions so failure on one meeting won't break a chain of + # tasks. This is used by agenda_data_refresh_all_task(). + log.log(f"ERROR: Refreshing agenda data failed for num={num}: {err}") @shared_task def agenda_data_refresh(): - generate_agenda_data(force_refresh=True) + """Deprecated. Use agenda_data_refresh_task() instead. + + TODO remove this after switching the periodic task to the new name + """ + log.log("Deprecated agenda_data_refresh task called!") + agenda_data_refresh_task() + + +@shared_task +def agenda_data_refresh_all_task(*, batch_size=10): + """Refresh agenda data for all plenary meetings + + Executes as a chain of tasks, each computing up to `batch_size` meetings + in a single task. + """ + meeting_numbers = sorted( + Meeting.objects.annotate( + number_as_int=Cast("number", output_field=IntegerField()) + ) + .filter(type_id="ietf", number_as_int__gt=64) + .values_list("number_as_int", flat=True) + ) + # Batch using chained maps rather than celery.chunk so we only use one worker + # at a time. + batched_task_chain = chain( + *( + agenda_data_refresh_task.map(nums) + for nums in batched(meeting_numbers, batch_size) + ) + ) + batched_task_chain.delay() @shared_task @@ -42,3 +100,149 @@ def proceedings_content_refresh_task(*, all=False): elif all or (num % 24 == now.hour): log.log(f"Refreshing proceedings for meeting {meeting.number}...") generate_proceedings_content(meeting, force_refresh=True) + + +@shared_task +def fetch_meeting_attendance_task(): + # fetch most recent two meetings + meetings = Meeting.objects.filter(type="ietf", date__lte=timezone.now()).order_by( + "-date" + )[:2] + try: + stats = fetch_attendance_from_meetings(meetings) + except RuntimeError as err: + log.log(f"Error in fetch_meeting_attendance_task: {err}") + else: + for meeting, meeting_stats in zip(meetings, stats): + log.log( + "Fetched data for meeting {:>3}: {:4d} created, {:4d} updated, {:4d} deleted, {:4d} processed".format( + meeting.number, + meeting_stats["created"], + meeting_stats["updated"], + meeting_stats["deleted"], + meeting_stats["processed"], + ) + ) + + +def _select_meetings( + meetings: list[str] | None = None, + meetings_since: str | None = None, + meetings_until: str | None = None, +): # nyah + """Select meetings by number or date range""" + # IETF-1 = 1986-01-16 + EARLIEST_MEETING_DATE = datetime.datetime(1986, 1, 1) + meetings_since_dt: datetime.datetime | None = None + meetings_until_dt: datetime.datetime | None = None + + if meetings_since == "zero": + meetings_since_dt = EARLIEST_MEETING_DATE + elif meetings_since is not None: + try: + meetings_since_dt = datetime.datetime.fromisoformat(meetings_since) + except ValueError: + log.log( + "Failed to parse meetings_since='{meetings_since}' with fromisoformat" + ) + raise + + if meetings_until is not None: + try: + meetings_until_dt = datetime.datetime.fromisoformat(meetings_until) + except ValueError: + log.log( + "Failed to parse meetings_until='{meetings_until}' with fromisoformat" + ) + raise + if meetings_since_dt is None: + # if we only got meetings_until, start from the first meeting + meetings_since_dt = EARLIEST_MEETING_DATE + + if meetings is None: + if meetings_since_dt is None: + log.log("No meetings requested, doing nothing.") + return Meeting.objects.none() + meetings_qs = Meeting.objects.filter(date__gte=meetings_since_dt) + if meetings_until_dt is not None: + meetings_qs = meetings_qs.filter(date__lte=meetings_until_dt) + log.log( + "Selecting meetings between " + f"{meetings_since_dt} and {meetings_until_dt}" + ) + else: + log.log(f"Selecting meetings since {meetings_since_dt}") + else: + if meetings_since_dt is not None: + log.log( + "Ignoring meetings_since and meetings_until " + "because specific meetings were requested." + ) + meetings_qs = Meeting.objects.filter(number__in=meetings) + return meetings_qs + + +@shared_task +def resolve_meeting_materials_task( + *, # only allow kw arguments + meetings: list[str] | None = None, + meetings_since: str | None = None, + meetings_until: str | None = None, +): + """Run materials resolver on meetings + + Can request a set of meetings by number by passing a list in the meetings arg, or + by range by passing an iso-format timestamps in meetings_since / meetings_until. + To select all meetings, set meetings_since="zero" and omit other parameters. + """ + meetings_qs = _select_meetings(meetings, meetings_since, meetings_until) + for meeting in meetings_qs.order_by("date"): + log.log( + f"Resolving materials for {meeting.type_id} " + f"meeting {meeting.number} ({meeting.date})..." + ) + mark = timezone.now() + try: + resolve_materials_for_one_meeting(meeting) + except Exception as err: + log.log( + "Exception raised while resolving materials for " + f"meeting {meeting.number}: {err}" + ) + else: + log.log( + f"Resolved in {(timezone.now() - mark).total_seconds():0.3f} seconds." + ) + + +@shared_task +def store_meeting_materials_as_blobs_task( + *, # only allow kw arguments + meetings: list[str] | None = None, + meetings_since: str | None = None, + meetings_until: str | None = None, +): + """Push meeting materials into the blob store + + Can request a set of meetings by number by passing a list in the meetings arg, or + by range by passing an iso-format timestamps in meetings_since / meetings_until. + To select all meetings, set meetings_since="zero" and omit other parameters. + """ + meetings_qs = _select_meetings(meetings, meetings_since, meetings_until) + for meeting in meetings_qs.order_by("date"): + log.log( + f"Creating blobs for materials for {meeting.type_id} " + f"meeting {meeting.number} ({meeting.date})..." + ) + mark = timezone.now() + try: + store_blobs_for_one_meeting(meeting) + except Exception as err: + log.log( + "Exception raised while creating blobs for " + f"meeting {meeting.number}: {err}" + ) + else: + log.log( + f"Blobs created in {(timezone.now() - mark).total_seconds():0.3f} seconds." + ) diff --git a/ietf/secr/sreq/templatetags/ams_filters.py b/ietf/meeting/templatetags/ams_filters.py similarity index 96% rename from ietf/secr/sreq/templatetags/ams_filters.py rename to ietf/meeting/templatetags/ams_filters.py index 3ef872232a..a8175a81d6 100644 --- a/ietf/secr/sreq/templatetags/ams_filters.py +++ b/ietf/meeting/templatetags/ams_filters.py @@ -1,3 +1,5 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + from django import template from ietf.person.models import Person diff --git a/ietf/meeting/tests_js.py b/ietf/meeting/tests_js.py index a184a7c6d0..3269342924 100644 --- a/ietf/meeting/tests_js.py +++ b/ietf/meeting/tests_js.py @@ -1260,7 +1260,7 @@ def _assert_ietf_tz_correct(meetings, tz): # to inherit Django's settings.TIME_ZONE but I don't know whether that's guaranteed to be consistent. # To avoid test fragility, ask Moment what it considers local and expect that. local_tz = self.driver.execute_script('return moment.tz.guess();') - local_tz_opt = tz_select_input.find_element(By.CSS_SELECTOR, 'option[value=%s]' % local_tz) + local_tz_opt = tz_select_input.find_element(By.CSS_SELECTOR, 'option[value="%s"]' % local_tz) local_tz_bottom_opt = tz_select_bottom_input.find_element(By.CSS_SELECTOR, 'option[value="%s"]' % local_tz) # Should start off in local time zone @@ -1576,7 +1576,7 @@ def test_delete_timeslot_cancel(self): def do_delete_time_interval_test(self, cancel=False): delete_time_local = datetime_from_date(self.meeting.date, self.meeting.tz()).replace(hour=10) - delete_time = delete_time_local.astimezone(datetime.timezone.utc) + delete_time = delete_time_local.astimezone(datetime.UTC) duration = datetime.timedelta(minutes=60) delete: [TimeSlot] = TimeSlotFactory.create_batch( # type: ignore[annotation-unchecked] diff --git a/ietf/meeting/tests_models.py b/ietf/meeting/tests_models.py index 03b706e1d7..869d9ec814 100644 --- a/ietf/meeting/tests_models.py +++ b/ietf/meeting/tests_models.py @@ -3,7 +3,7 @@ """Tests of models in the Meeting application""" import datetime -from mock import patch +from unittest.mock import patch from django.conf import settings from django.test import override_settings @@ -11,8 +11,8 @@ import ietf.meeting.models from ietf.group.factories import GroupFactory, GroupHistoryFactory from ietf.meeting.factories import MeetingFactory, SessionFactory, AttendedFactory, SessionPresentationFactory +from ietf.meeting.factories import RegistrationFactory from ietf.meeting.models import Session -from ietf.stats.factories import MeetingRegistrationFactory from ietf.utils.test_utils import TestCase from ietf.utils.timezone import date_today, datetime_today @@ -21,9 +21,9 @@ class MeetingTests(TestCase): def test_get_attendance_pre110(self): """Pre-110 meetings do not calculate attendance""" meeting = MeetingFactory(type_id='ietf', number='109') - MeetingRegistrationFactory.create_batch(3, meeting=meeting, reg_type='') - MeetingRegistrationFactory.create_batch(4, meeting=meeting, reg_type='remote') - MeetingRegistrationFactory.create_batch(5, meeting=meeting, reg_type='in_person') + RegistrationFactory.create_batch(3, meeting=meeting, with_ticket={'attendance_type_id': 'unknown'}) + RegistrationFactory.create_batch(4, meeting=meeting, with_ticket={'attendance_type_id': 'remote'}) + RegistrationFactory.create_batch(5, meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}) self.assertIsNone(meeting.get_attendance()) def test_get_attendance_110(self): @@ -31,31 +31,31 @@ def test_get_attendance_110(self): meeting = MeetingFactory(type_id='ietf', number='110') # start with attendees that should be ignored - MeetingRegistrationFactory.create_batch(3, meeting=meeting, reg_type='', attended=True) - MeetingRegistrationFactory(meeting=meeting, reg_type='', attended=False) + RegistrationFactory.create_batch(3, meeting=meeting, with_ticket={'attendance_type_id': 'unknown'}, attended=True) + RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'unknown'}, attended=False) attendance = meeting.get_attendance() self.assertIsNotNone(attendance) self.assertEqual(attendance.remote, 0) self.assertEqual(attendance.onsite, 0) # add online attendees with at least one who registered but did not attend - MeetingRegistrationFactory.create_batch(4, meeting=meeting, reg_type='remote', attended=True) - MeetingRegistrationFactory(meeting=meeting, reg_type='remote', attended=False) + RegistrationFactory.create_batch(4, meeting=meeting, with_ticket={'attendance_type_id': 'remote'}, attended=True) + RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'remote'}, attended=False) attendance = meeting.get_attendance() self.assertIsNotNone(attendance) self.assertEqual(attendance.remote, 4) self.assertEqual(attendance.onsite, 0) # and the same for onsite attendees - MeetingRegistrationFactory.create_batch(5, meeting=meeting, reg_type='onsite', attended=True) - MeetingRegistrationFactory(meeting=meeting, reg_type='in_person', attended=False) + RegistrationFactory.create_batch(5, meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=True) + RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=False) attendance = meeting.get_attendance() self.assertIsNotNone(attendance) self.assertEqual(attendance.remote, 4) self.assertEqual(attendance.onsite, 5) # and once more after removing all the online attendees - meeting.meetingregistration_set.filter(reg_type='remote').delete() + meeting.registration_set.remote().delete() attendance = meeting.get_attendance() self.assertIsNotNone(attendance) self.assertEqual(attendance.remote, 0) @@ -64,11 +64,11 @@ def test_get_attendance_110(self): def test_get_attendance_113(self): """Simulate IETF 113 attendance gathering data""" meeting = MeetingFactory(type_id='ietf', number='113') - MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', attended=True, checkedin=False) - MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', attended=False, checkedin=True) - p1 = MeetingRegistrationFactory(meeting=meeting, reg_type='onsite', attended=False, checkedin=False).person + RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=True, checkedin=False) + RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=False, checkedin=True) + p1 = RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'onsite'}, attended=False, checkedin=False).person AttendedFactory(session__meeting=meeting, person=p1) - p2 = MeetingRegistrationFactory(meeting=meeting, reg_type='remote', attended=False, checkedin=False).person + p2 = RegistrationFactory(meeting=meeting, with_ticket={'attendance_type_id': 'remote'}, attended=False, checkedin=False).person AttendedFactory(session__meeting=meeting, person=p2) attendance = meeting.get_attendance() self.assertEqual(attendance.onsite, 3) @@ -82,9 +82,9 @@ def test_get_attendance_keeps_meetings_distinct(self): # Create a person who attended a remote session for first_mtg and onsite for second_mtg without # checking in for either. - p = MeetingRegistrationFactory(meeting=second_mtg, reg_type='onsite', attended=False, checkedin=False).person + p = RegistrationFactory(meeting=second_mtg, with_ticket={'attendance_type_id': 'onsite'}, attended=False, checkedin=False).person AttendedFactory(session__meeting=first_mtg, person=p) - MeetingRegistrationFactory(meeting=first_mtg, person=p, reg_type='remote', attended=False, checkedin=False) + RegistrationFactory(meeting=first_mtg, person=p, with_ticket={'attendance_type_id': 'remote'}, attended=False, checkedin=False) AttendedFactory(session__meeting=second_mtg, person=p) att = first_mtg.get_attendance() diff --git a/ietf/secr/sreq/tests.py b/ietf/meeting/tests_session_requests.py similarity index 84% rename from ietf/secr/sreq/tests.py rename to ietf/meeting/tests_session_requests.py index 847b993e1c..42dbee5f23 100644 --- a/ietf/secr/sreq/tests.py +++ b/ietf/meeting/tests_session_requests.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2013-2022, All Rights Reserved +# Copyright The IETF Trust 2013-2025, All Rights Reserved # -*- coding: utf-8 -*- @@ -15,30 +15,15 @@ from ietf.name.models import ConstraintName, TimerangeName from ietf.person.factories import PersonFactory from ietf.person.models import Person -from ietf.secr.sreq.forms import SessionForm +from ietf.meeting.forms import SessionRequestForm from ietf.utils.mail import outbox, empty_outbox, get_payload_text, send_mail from ietf.utils.timezone import date_today from pyquery import PyQuery -SECR_USER='secretary' +SECR_USER = 'secretary' -class SreqUrlTests(TestCase): - def test_urls(self): - MeetingFactory(type_id='ietf',date=date_today()) - - self.client.login(username="secretary", password="secretary+password") - - r = self.client.get("/secr/") - self.assertEqual(r.status_code, 200) - - r = self.client.get("/secr/sreq/") - self.assertEqual(r.status_code, 200) - - testgroup=GroupFactory() - r = self.client.get("/secr/sreq/%s/new/" % testgroup.acronym) - self.assertEqual(r.status_code, 200) class SessionRequestTestCase(TestCase): def test_main(self): @@ -46,7 +31,7 @@ def test_main(self): SessionFactory.create_batch(2, meeting=meeting, status_id='sched') SessionFactory.create_batch(2, meeting=meeting, status_id='disappr') # Several unscheduled groups come from make_immutable_base_data - url = reverse('ietf.secr.sreq.views.main') + url = reverse('ietf.meeting.views_session_request.list_view') self.client.login(username="secretary", password="secretary+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) @@ -62,27 +47,27 @@ def test_approve(self): mars = GroupFactory(parent=area, acronym='mars') # create session waiting for approval session = SessionFactory(meeting=meeting, group=mars, status_id='apprw') - url = reverse('ietf.secr.sreq.views.approve', kwargs={'acronym':'mars'}) + url = reverse('ietf.meeting.views_session_request.approve_request', kwargs={'acronym': 'mars'}) self.client.login(username="ad", password="ad+password") r = self.client.get(url) - self.assertRedirects(r,reverse('ietf.secr.sreq.views.view', kwargs={'acronym':'mars'})) + self.assertRedirects(r, reverse('ietf.meeting.views_session_request.view_request', kwargs={'acronym': 'mars'})) self.assertEqual(SchedulingEvent.objects.filter(session=session).order_by('-id')[0].status_id, 'appr') - + def test_cancel(self): meeting = MeetingFactory(type_id='ietf', date=date_today()) ad = Person.objects.get(user__username='ad') area = RoleFactory(name_id='ad', person=ad, group__type_id='area').group session = SessionFactory(meeting=meeting, group__parent=area, group__acronym='mars', status_id='sched') - url = reverse('ietf.secr.sreq.views.cancel', kwargs={'acronym':'mars'}) + url = reverse('ietf.meeting.views_session_request.cancel_request', kwargs={'acronym': 'mars'}) self.client.login(username="ad", password="ad+password") r = self.client.get(url) - self.assertRedirects(r,reverse('ietf.secr.sreq.views.main')) + self.assertRedirects(r, reverse('ietf.meeting.views_session_request.list_view')) self.assertEqual(SchedulingEvent.objects.filter(session=session).order_by('-id')[0].status_id, 'deleted') def test_cancel_notification_msg(self): to = "" subject = "Dummy subject" - template = "sreq/session_cancel_notification.txt" + template = "meeting/session_cancel_notification.txt" meeting = MeetingFactory(type_id="ietf", date=date_today()) requester = PersonFactory(name="James O'Rourke", user__username="jimorourke") context = {"meeting": meeting, "requester": requester} @@ -113,9 +98,9 @@ def test_edit(self): group4 = GroupFactory() iabprog = GroupFactory(type_id='program') - SessionFactory(meeting=meeting,group=mars,status_id='sched') + SessionFactory(meeting=meeting, group=mars, status_id='sched') - url = reverse('ietf.secr.sreq.views.edit', kwargs={'acronym':'mars'}) + url = reverse('ietf.meeting.views_session_request.edit_request', kwargs={'acronym': 'mars'}) self.client.login(username="marschairman", password="marschairman+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) @@ -123,9 +108,9 @@ def test_edit(self): comments = 'need lights' mars_sessions = meeting.session_set.filter(group__acronym='mars') empty_outbox() - post_data = {'num_session':'2', + post_data = {'num_session': '2', 'attendees': attendees, - 'constraint_chair_conflict':iabprog.acronym, + 'constraint_chair_conflict': iabprog.acronym, 'session_time_relation': 'subsequent-days', 'adjacent_with_wg': group2.acronym, 'joint_with_groups': group3.acronym + ' ' + group4.acronym, @@ -135,7 +120,7 @@ def test_edit(self): 'session_set-INITIAL_FORMS': '1', 'session_set-MIN_NUM_FORMS': '1', 'session_set-MAX_NUM_FORMS': '3', - 'session_set-0-id':mars_sessions[0].pk, + 'session_set-0-id': mars_sessions[0].pk, 'session_set-0-name': mars_sessions[0].name, 'session_set-0-short': mars_sessions[0].short, 'session_set-0-purpose': mars_sessions[0].purpose_id, @@ -169,7 +154,7 @@ def test_edit(self): 'session_set-2-DELETE': 'on', 'submit': 'Continue'} r = self.client.post(url, post_data, HTTP_HOST='example.com') - redirect_url = reverse('ietf.secr.sreq.views.view', kwargs={'acronym': 'mars'}) + redirect_url = reverse('ietf.meeting.views_session_request.view_request', kwargs={'acronym': 'mars'}) self.assertRedirects(r, redirect_url) # Check whether updates were stored in the database @@ -204,17 +189,17 @@ def test_edit(self): # Edit again, changing the joint sessions and clearing some fields. The behaviour of # edit is different depending on whether previous joint sessions were recorded. empty_outbox() - post_data = {'num_session':'2', - 'attendees':attendees, - 'constraint_chair_conflict':'', - 'comments':'need lights', + post_data = {'num_session': '2', + 'attendees': attendees, + 'constraint_chair_conflict': '', + 'comments': 'need lights', 'joint_with_groups': group2.acronym, 'joint_for_session': '1', 'session_set-TOTAL_FORMS': '3', # matches what view actually sends, even with only 2 filled in 'session_set-INITIAL_FORMS': '2', 'session_set-MIN_NUM_FORMS': '1', 'session_set-MAX_NUM_FORMS': '3', - 'session_set-0-id':sessions[0].pk, + 'session_set-0-id': sessions[0].pk, 'session_set-0-name': sessions[0].name, 'session_set-0-short': sessions[0].short, 'session_set-0-purpose': sessions[0].purpose_id, @@ -251,7 +236,7 @@ def test_edit(self): self.assertRedirects(r, redirect_url) # Check whether updates were stored in the database - sessions = Session.objects.filter(meeting=meeting, group=mars) + sessions = Session.objects.filter(meeting=meeting, group=mars).order_by("id") self.assertEqual(len(sessions), 2) session = sessions[0] self.assertFalse(session.constraints().filter(name='time_relation')) @@ -270,7 +255,6 @@ def test_edit(self): r = self.client.get(redirect_url) self.assertContains(r, 'First session with: {}'.format(group2.acronym)) - def test_edit_constraint_bethere(self): meeting = MeetingFactory(type_id='ietf', date=date_today()) mars = RoleFactory(name_id='chair', person__user__username='marschairman', group__acronym='mars').group @@ -282,7 +266,7 @@ def test_edit_constraint_bethere(self): name_id='bethere', ) self.assertEqual(session.people_constraints.count(), 1) - url = reverse('ietf.secr.sreq.views.edit', kwargs=dict(acronym='mars')) + url = reverse('ietf.meeting.views_session_request.edit_request', kwargs=dict(acronym='mars')) self.client.login(username='marschairman', password='marschairman+password') attendees = '10' ad = Person.objects.get(user__username='ad') @@ -290,8 +274,8 @@ def test_edit_constraint_bethere(self): 'num_session': '1', 'attendees': attendees, 'bethere': str(ad.pk), - 'constraint_chair_conflict':'', - 'comments':'', + 'constraint_chair_conflict': '', + 'comments': '', 'joint_with_groups': '', 'joint_for_session': '', 'delete_conflict': 'on', @@ -299,7 +283,7 @@ def test_edit_constraint_bethere(self): 'session_set-INITIAL_FORMS': '1', 'session_set-MIN_NUM_FORMS': '1', 'session_set-MAX_NUM_FORMS': '3', - 'session_set-0-id':session.pk, + 'session_set-0-id': session.pk, 'session_set-0-name': session.name, 'session_set-0-short': session.short, 'session_set-0-purpose': session.purpose_id, @@ -313,8 +297,8 @@ def test_edit_constraint_bethere(self): 'session_set-1-id': '', 'session_set-1-name': '', 'session_set-1-short': '', - 'session_set-1-purpose':'regular', - 'session_set-1-type':'regular', + 'session_set-1-purpose': 'regular', + 'session_set-1-type': 'regular', 'session_set-1-requested_duration': '', 'session_set-1-on_agenda': 'True', 'session_set-1-attendees': attendees, @@ -333,7 +317,7 @@ def test_edit_constraint_bethere(self): 'submit': 'Save', } r = self.client.post(url, post_data, HTTP_HOST='example.com') - redirect_url = reverse('ietf.secr.sreq.views.view', kwargs={'acronym': 'mars'}) + redirect_url = reverse('ietf.meeting.views_session_request.view_request', kwargs={'acronym': 'mars'}) self.assertRedirects(r, redirect_url) self.assertEqual([pc.person for pc in session.people_constraints.all()], [ad]) @@ -350,7 +334,7 @@ def test_edit_inactive_conflicts(self): target=other_group, ) - url = reverse('ietf.secr.sreq.views.edit', kwargs=dict(acronym='mars')) + url = reverse('ietf.meeting.views_session_request.edit_request', kwargs=dict(acronym='mars')) self.client.login(username='marschairman', password='marschairman+password') r = self.client.get(url) self.assertEqual(r.status_code, 200) @@ -360,17 +344,17 @@ def test_edit_inactive_conflicts(self): found = q('input#id_delete_conflict[type="checkbox"]') self.assertEqual(len(found), 1) delete_checkbox = found[0] - # check that the label on the checkbox is correct - self.assertIn('Delete this conflict', delete_checkbox.tail) + self.assertIn('Delete this conflict', delete_checkbox.label.text) # check that the target is displayed correctly in the UI - self.assertIn(other_group.acronym, delete_checkbox.find('../input[@type="text"]').value) + row = found.parent().parent() + self.assertIn(other_group.acronym, row.find('input[@type="text"]').val()) attendees = '10' post_data = { 'num_session': '1', 'attendees': attendees, - 'constraint_chair_conflict':'', - 'comments':'', + 'constraint_chair_conflict': '', + 'comments': '', 'joint_with_groups': '', 'joint_for_session': '', 'delete_conflict': 'on', @@ -378,7 +362,7 @@ def test_edit_inactive_conflicts(self): 'session_set-INITIAL_FORMS': '1', 'session_set-MIN_NUM_FORMS': '1', 'session_set-MAX_NUM_FORMS': '3', - 'session_set-0-id':session.pk, + 'session_set-0-id': session.pk, 'session_set-0-name': session.name, 'session_set-0-short': session.short, 'session_set-0-purpose': session.purpose_id, @@ -392,28 +376,28 @@ def test_edit_inactive_conflicts(self): 'submit': 'Save', } r = self.client.post(url, post_data, HTTP_HOST='example.com') - redirect_url = reverse('ietf.secr.sreq.views.view', kwargs={'acronym': 'mars'}) + redirect_url = reverse('ietf.meeting.views_session_request.view_request', kwargs={'acronym': 'mars'}) self.assertRedirects(r, redirect_url) self.assertEqual(len(mars.constraint_source_set.filter(name_id='conflict')), 0) def test_tool_status(self): MeetingFactory(type_id='ietf', date=date_today()) - url = reverse('ietf.secr.sreq.views.tool_status') + url = reverse('ietf.meeting.views_session_request.status') self.client.login(username="secretary", password="secretary+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) - r = self.client.post(url, {'message':'locked', 'submit':'Lock'}) - self.assertRedirects(r,reverse('ietf.secr.sreq.views.main')) + r = self.client.post(url, {'message': 'locked', 'submit': 'Lock'}) + self.assertRedirects(r, reverse('ietf.meeting.views_session_request.list_view')) def test_new_req_constraint_types(self): """Configurable constraint types should be handled correctly in a new request - Relies on SessionForm representing constraint values with element IDs + Relies on SessionRequestForm representing constraint values with element IDs like id_constraint_ """ meeting = MeetingFactory(type_id='ietf', date=date_today()) RoleFactory(name_id='chair', person__user__username='marschairman', group__acronym='mars') - url = reverse('ietf.secr.sreq.views.new', kwargs=dict(acronym='mars')) + url = reverse('ietf.meeting.views_session_request.new_request', kwargs=dict(acronym='mars')) self.client.login(username="marschairman", password="marschairman+password") for expected in [ @@ -441,7 +425,7 @@ def test_edit_req_constraint_types(self): add_to_schedule=False) RoleFactory(name_id='chair', person__user__username='marschairman', group__acronym='mars') - url = reverse('ietf.secr.sreq.views.edit', kwargs=dict(acronym='mars')) + url = reverse('ietf.meeting.views_session_request.edit_request', kwargs=dict(acronym='mars')) self.client.login(username='marschairman', password='marschairman+password') for expected in [ @@ -460,6 +444,7 @@ def test_edit_req_constraint_types(self): ['id_constraint_{}'.format(conf_name) for conf_name in expected], ) + class SubmitRequestCase(TestCase): def setUp(self): super(SubmitRequestCase, self).setUp() @@ -476,15 +461,15 @@ def test_submit_request(self): group3 = GroupFactory(parent=area) group4 = GroupFactory(parent=area) session_count_before = Session.objects.filter(meeting=meeting, group=group).count() - url = reverse('ietf.secr.sreq.views.new',kwargs={'acronym':group.acronym}) - confirm_url = reverse('ietf.secr.sreq.views.confirm',kwargs={'acronym':group.acronym}) - main_url = reverse('ietf.secr.sreq.views.main') + url = reverse('ietf.meeting.views_session_request.new_request', kwargs={'acronym': group.acronym}) + confirm_url = reverse('ietf.meeting.views_session_request.confirm', kwargs={'acronym': group.acronym}) + main_url = reverse('ietf.meeting.views_session_request.list_view') attendees = '10' comments = 'need projector' - post_data = {'num_session':'1', - 'attendees':attendees, - 'constraint_chair_conflict':'', - 'comments':comments, + post_data = {'num_session': '1', + 'attendees': attendees, + 'constraint_chair_conflict': '', + 'comments': comments, 'adjacent_with_wg': group2.acronym, 'timeranges': ['thursday-afternoon-early', 'thursday-afternoon-late'], 'joint_with_groups': group3.acronym + ' ' + group4.acronym, @@ -506,7 +491,7 @@ def test_submit_request(self): 'session_set-0-DELETE': '', 'submit': 'Continue'} self.client.login(username="secretary", password="secretary+password") - r = self.client.post(url,post_data) + r = self.client.post(url, post_data) self.assertEqual(r.status_code, 200) # Verify the contents of the confirm view @@ -515,13 +500,13 @@ def test_submit_request(self): self.assertContains(r, 'First session with: {} {}'.format(group3.acronym, group4.acronym)) post_data['submit'] = 'Submit' - r = self.client.post(confirm_url,post_data) + r = self.client.post(confirm_url, post_data) self.assertRedirects(r, main_url) session_count_after = Session.objects.filter(meeting=meeting, group=group, type='regular').count() self.assertEqual(session_count_after, session_count_before + 1) # test that second confirm does not add sessions - r = self.client.post(confirm_url,post_data) + r = self.client.post(confirm_url, post_data) self.assertRedirects(r, main_url) session_count_after = Session.objects.filter(meeting=meeting, group=group, type='regular').count() self.assertEqual(session_count_after, session_count_before + 1) @@ -535,42 +520,6 @@ def test_submit_request(self): ) self.assertEqual(set(list(session.joint_with_groups.all())), set([group3, group4])) - def test_submit_request_invalid(self): - MeetingFactory(type_id='ietf', date=date_today()) - ad = Person.objects.get(user__username='ad') - area = RoleFactory(name_id='ad', person=ad, group__type_id='area').group - group = GroupFactory(parent=area) - url = reverse('ietf.secr.sreq.views.new',kwargs={'acronym':group.acronym}) - attendees = '10' - comments = 'need projector' - post_data = { - 'num_session':'2', - 'attendees':attendees, - 'constraint_chair_conflict':'', - 'comments':comments, - 'session_set-TOTAL_FORMS': '1', - 'session_set-INITIAL_FORMS': '1', - 'session_set-MIN_NUM_FORMS': '1', - 'session_set-MAX_NUM_FORMS': '3', - # no 'session_set-0-id' to create a new session - 'session_set-0-name': '', - 'session_set-0-short': '', - 'session_set-0-purpose': 'regular', - 'session_set-0-type': 'regular', - 'session_set-0-requested_duration': '3600', - 'session_set-0-on_agenda': True, - 'session_set-0-remote_instructions': '', - 'session_set-0-attendees': attendees, - 'session_set-0-comments': comments, - 'session_set-0-DELETE': '', - } - self.client.login(username="secretary", password="secretary+password") - r = self.client.post(url,post_data) - self.assertEqual(r.status_code, 200) - q = PyQuery(r.content) - self.assertEqual(len(q('#session-request-form')),1) - self.assertContains(r, 'Must provide data for all sessions') - def test_submit_request_check_constraints(self): m1 = MeetingFactory(type_id='ietf', date=date_today() - datetime.timedelta(days=100)) MeetingFactory(type_id='ietf', date=date_today(), @@ -597,7 +546,7 @@ def test_submit_request_check_constraints(self): self.client.login(username="secretary", password="secretary+password") - url = reverse('ietf.secr.sreq.views.new',kwargs={'acronym':group.acronym}) + url = reverse('ietf.meeting.views_session_request.new_request', kwargs={'acronym': group.acronym}) r = self.client.get(url + '?previous') self.assertEqual(r.status_code, 200) q = PyQuery(r.content) @@ -607,11 +556,11 @@ def test_submit_request_check_constraints(self): attendees = '10' comments = 'need projector' - post_data = {'num_session':'1', - 'attendees':attendees, + post_data = {'num_session': '1', + 'attendees': attendees, 'constraint_chair_conflict': group.acronym, - 'comments':comments, - 'session_set-TOTAL_FORMS': '1', + 'comments': comments, + 'session_set-TOTAL_FORMS': '3', 'session_set-INITIAL_FORMS': '1', 'session_set-MIN_NUM_FORMS': '1', 'session_set-MAX_NUM_FORMS': '3', @@ -626,11 +575,31 @@ def test_submit_request_check_constraints(self): 'session_set-0-attendees': attendees, 'session_set-0-comments': comments, 'session_set-0-DELETE': '', + 'session_set-1-name': '', + 'session_set-1-short': '', + 'session_set-1-purpose': session.purpose_id, + 'session_set-1-type': session.type_id, + 'session_set-1-requested_duration': '', + 'session_set-1-on_agenda': session.on_agenda, + 'session_set-1-remote_instructions': '', + 'session_set-1-attendees': attendees, + 'session_set-1-comments': '', + 'session_set-1-DELETE': 'on', + 'session_set-2-name': '', + 'session_set-2-short': '', + 'session_set-2-purpose': session.purpose_id, + 'session_set-2-type': session.type_id, + 'session_set-2-requested_duration': '', + 'session_set-2-on_agenda': session.on_agenda, + 'session_set-2-remote_instructions': '', + 'session_set-2-attendees': attendees, + 'session_set-2-comments': '', + 'session_set-2-DELETE': 'on', 'submit': 'Continue'} - r = self.client.post(url,post_data) + r = self.client.post(url, post_data) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertEqual(len(q('#session-request-form')),1) + self.assertEqual(len(q('#session-request-form')), 1) self.assertContains(r, "Cannot declare a conflict with the same group") def test_request_notification(self): @@ -645,18 +614,18 @@ def test_request_notification(self): RoleFactory(name_id='chair', group=group, person__user__username='ameschairman') resource = ResourceAssociation.objects.create(name_id='project') # Bit of a test data hack - the fixture now has no used resources to pick from - resource.name.used=True + resource.name.used = True resource.name.save() - url = reverse('ietf.secr.sreq.views.new',kwargs={'acronym':group.acronym}) - confirm_url = reverse('ietf.secr.sreq.views.confirm',kwargs={'acronym':group.acronym}) + url = reverse('ietf.meeting.views_session_request.new_request', kwargs={'acronym': group.acronym}) + confirm_url = reverse('ietf.meeting.views_session_request.confirm', kwargs={'acronym': group.acronym}) len_before = len(outbox) attendees = '10' - post_data = {'num_session':'2', - 'attendees':attendees, - 'bethere':str(ad.pk), - 'constraint_chair_conflict':group4.acronym, - 'comments':'', + post_data = {'num_session': '2', + 'attendees': attendees, + 'bethere': str(ad.pk), + 'constraint_chair_conflict': group4.acronym, + 'comments': '', 'resources': resource.pk, 'session_time_relation': 'subsequent-days', 'adjacent_with_wg': group2.acronym, @@ -692,23 +661,23 @@ def test_request_notification(self): 'submit': 'Continue'} self.client.login(username="ameschairman", password="ameschairman+password") # submit - r = self.client.post(url,post_data) + r = self.client.post(url, post_data) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertTrue('Confirm' in str(q("title")), r.context['form'].errors) # confirm post_data['submit'] = 'Submit' - r = self.client.post(confirm_url,post_data) - self.assertRedirects(r, reverse('ietf.secr.sreq.views.main')) - self.assertEqual(len(outbox),len_before+1) + r = self.client.post(confirm_url, post_data) + self.assertRedirects(r, reverse('ietf.meeting.views_session_request.list_view')) + self.assertEqual(len(outbox), len_before + 1) notification = outbox[-1] notification_payload = get_payload_text(notification) - sessions = Session.objects.filter(meeting=meeting,group=group) + sessions = Session.objects.filter(meeting=meeting, group=group) self.assertEqual(len(sessions), 2) session = sessions[0] - self.assertEqual(session.resources.count(),1) - self.assertEqual(session.people_constraints.count(),1) + self.assertEqual(session.resources.count(), 1) + self.assertEqual(session.people_constraints.count(), 1) self.assertEqual(session.constraints().get(name='time_relation').time_relation, 'subsequent-days') self.assertEqual(session.constraints().get(name='wg_adjacent').target.acronym, group2.acronym) self.assertEqual( @@ -731,7 +700,7 @@ def test_request_notification(self): def test_request_notification_msg(self): to = "" subject = "Dummy subject" - template = "sreq/session_request_notification.txt" + template = "meeting/session_request_notification.txt" header = "A new" meeting = MeetingFactory(type_id="ietf", date=date_today()) requester = PersonFactory(name="James O'Rourke", user__username="jimorourke") @@ -767,19 +736,19 @@ def test_request_notification_third_session(self): RoleFactory(name_id='chair', group=group, person__user__username='ameschairman') resource = ResourceAssociation.objects.create(name_id='project') # Bit of a test data hack - the fixture now has no used resources to pick from - resource.name.used=True + resource.name.used = True resource.name.save() - url = reverse('ietf.secr.sreq.views.new',kwargs={'acronym':group.acronym}) - confirm_url = reverse('ietf.secr.sreq.views.confirm',kwargs={'acronym':group.acronym}) + url = reverse('ietf.meeting.views_session_request.new_request', kwargs={'acronym': group.acronym}) + confirm_url = reverse('ietf.meeting.views_session_request.confirm', kwargs={'acronym': group.acronym}) len_before = len(outbox) attendees = '10' - post_data = {'num_session':'2', + post_data = {'num_session': '2', 'third_session': 'true', - 'attendees':attendees, - 'bethere':str(ad.pk), - 'constraint_chair_conflict':group4.acronym, - 'comments':'', + 'attendees': attendees, + 'bethere': str(ad.pk), + 'constraint_chair_conflict': group4.acronym, + 'comments': '', 'resources': resource.pk, 'session_time_relation': 'subsequent-days', 'adjacent_with_wg': group2.acronym, @@ -826,23 +795,23 @@ def test_request_notification_third_session(self): 'submit': 'Continue'} self.client.login(username="ameschairman", password="ameschairman+password") # submit - r = self.client.post(url,post_data) + r = self.client.post(url, post_data) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertTrue('Confirm' in str(q("title")), r.context['form'].errors) # confirm post_data['submit'] = 'Submit' - r = self.client.post(confirm_url,post_data) - self.assertRedirects(r, reverse('ietf.secr.sreq.views.main')) - self.assertEqual(len(outbox),len_before+1) + r = self.client.post(confirm_url, post_data) + self.assertRedirects(r, reverse('ietf.meeting.views_session_request.list_view')) + self.assertEqual(len(outbox), len_before + 1) notification = outbox[-1] notification_payload = get_payload_text(notification) - sessions = Session.objects.filter(meeting=meeting,group=group) + sessions = Session.objects.filter(meeting=meeting, group=group) self.assertEqual(len(sessions), 3) session = sessions[0] - self.assertEqual(session.resources.count(),1) - self.assertEqual(session.people_constraints.count(),1) + self.assertEqual(session.resources.count(), 1) + self.assertEqual(session.people_constraints.count(), 1) self.assertEqual(session.constraints().get(name='time_relation').time_relation, 'subsequent-days') self.assertEqual(session.constraints().get(name='wg_adjacent').target.acronym, group2.acronym) self.assertEqual( @@ -861,16 +830,17 @@ def test_request_notification_third_session(self): self.assertIn('1 Hour, 1 Hour, 1 Hour', notification_payload) self.assertIn('The third session requires your approval', notification_payload) + class LockAppTestCase(TestCase): def setUp(self): super().setUp() - self.meeting = MeetingFactory(type_id='ietf', date=date_today(),session_request_lock_message='locked') + self.meeting = MeetingFactory(type_id='ietf', date=date_today(), session_request_lock_message='locked') self.group = GroupFactory(acronym='mars') RoleFactory(name_id='chair', group=self.group, person__user__username='marschairman') - SessionFactory(group=self.group,meeting=self.meeting) + SessionFactory(group=self.group, meeting=self.meeting) def test_edit_request(self): - url = reverse('ietf.secr.sreq.views.edit',kwargs={'acronym':self.group.acronym}) + url = reverse('ietf.meeting.views_session_request.edit_request', kwargs={'acronym': self.group.acronym}) self.client.login(username="secretary", password="secretary+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) @@ -882,48 +852,49 @@ def test_edit_request(self): self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertEqual(len(q(':disabled[name="submit"]')), 1) - + def test_view_request(self): - url = reverse('ietf.secr.sreq.views.view',kwargs={'acronym':self.group.acronym}) + url = reverse('ietf.meeting.views_session_request.view_request', kwargs={'acronym': self.group.acronym}) self.client.login(username="secretary", password="secretary+password") - r = self.client.get(url,follow=True) + r = self.client.get(url, follow=True) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertEqual(len(q(':enabled[name="edit"]')), 1) # secretary can edit chair = self.group.role_set.filter(name_id='chair').first().person.user.username self.client.login(username=chair, password=f'{chair}+password') - r = self.client.get(url,follow=True) + r = self.client.get(url, follow=True) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertEqual(len(q(':disabled[name="edit"]')), 1) # chair cannot edit def test_new_request(self): - url = reverse('ietf.secr.sreq.views.new',kwargs={'acronym':self.group.acronym}) - + url = reverse('ietf.meeting.views_session_request.new_request', kwargs={'acronym': self.group.acronym}) + # try as WG Chair self.client.login(username="marschairman", password="marschairman+password") r = self.client.get(url, follow=True) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertEqual(len(q('#session-request-form')),0) - + self.assertEqual(len(q('#session-request-form')), 0) + # try as Secretariat self.client.login(username="secretary", password="secretary+password") - r = self.client.get(url,follow=True) + r = self.client.get(url, follow=True) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertEqual(len(q('#session-request-form')),1) - + self.assertEqual(len(q('#session-request-form')), 1) + + class NotMeetingCase(TestCase): def test_not_meeting(self): - MeetingFactory(type_id='ietf',date=date_today()) + MeetingFactory(type_id='ietf', date=date_today()) group = GroupFactory(acronym='mars') - url = reverse('ietf.secr.sreq.views.no_session',kwargs={'acronym':group.acronym}) + url = reverse('ietf.meeting.views_session_request.no_session', kwargs={'acronym': group.acronym}) self.client.login(username="secretary", password="secretary+password") empty_outbox() - r = self.client.get(url,follow=True) + r = self.client.get(url, follow=True) # If the view invoked by that get throws an exception (such as an integrity error), # the traceback from this test will talk about a TransactionManagementError and # yell about executing queries before the end of an 'atomic' block @@ -932,14 +903,15 @@ def test_not_meeting(self): self.assertEqual(r.status_code, 200) self.assertContains(r, 'A message was sent to notify not having a session') - r = self.client.get(url,follow=True) + r = self.client.get(url, follow=True) self.assertEqual(r.status_code, 200) self.assertContains(r, 'is already marked as not meeting') - self.assertEqual(len(outbox),1) + self.assertEqual(len(outbox), 1) self.assertTrue('Not having a session' in outbox[0]['Subject']) self.assertTrue('session-request@' in outbox[0]['To']) + class RetrievePreviousCase(TestCase): pass @@ -949,7 +921,7 @@ class RetrievePreviousCase(TestCase): # test access by unauthorized -class SessionFormTest(TestCase): +class SessionRequestFormTest(TestCase): def setUp(self): super().setUp() self.meeting = MeetingFactory(type_id='ietf') @@ -1014,19 +986,19 @@ def setUp(self): 'session_set-2-comments': '', 'session_set-2-DELETE': '', } - + def test_valid(self): # Test with three sessions - form = SessionForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting) + form = SessionRequestForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting) self.assertTrue(form.is_valid()) - + # Test with two sessions self.valid_form_data.update({ 'third_session': '', 'session_set-TOTAL_FORMS': '2', 'joint_for_session': '2' }) - form = SessionForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting) + form = SessionRequestForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting) self.assertTrue(form.is_valid()) # Test with one session @@ -1036,9 +1008,9 @@ def test_valid(self): 'joint_for_session': '1', 'session_time_relation': '', }) - form = SessionForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting) + form = SessionRequestForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting) self.assertTrue(form.is_valid()) - + def test_invalid_groups(self): new_form_data = { 'constraint_chair_conflict': 'doesnotexist', @@ -1057,7 +1029,7 @@ def test_valid_group_appears_in_multiple_conflicts(self): 'constraint_tech_overlap': self.group2.acronym, } self.valid_form_data.update(new_form_data) - form = SessionForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting) + form = SessionRequestForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting) self.assertTrue(form.is_valid()) def test_invalid_group_appears_in_multiple_conflicts(self): @@ -1116,7 +1088,7 @@ def test_invalid_joint_for_session(self): 'joint_for_session': [ 'Session 2 can not be the joint session, the session has not been requested.'] }) - + def test_invalid_missing_session_length(self): form = self._invalid_test_helper({ 'session_set-TOTAL_FORMS': '2', @@ -1156,6 +1128,6 @@ def test_invalid_missing_session_length(self): def _invalid_test_helper(self, new_form_data): form_data = dict(self.valid_form_data, **new_form_data) - form = SessionForm(data=form_data, group=self.group1, meeting=self.meeting) + form = SessionRequestForm(data=form_data, group=self.group1, meeting=self.meeting) self.assertFalse(form.is_valid()) return form diff --git a/ietf/meeting/tests_tasks.py b/ietf/meeting/tests_tasks.py index c026a99835..2c5120a39d 100644 --- a/ietf/meeting/tests_tasks.py +++ b/ietf/meeting/tests_tasks.py @@ -1,27 +1,69 @@ # Copyright The IETF Trust 2025, All Rights Reserved import datetime -from mock import patch, call +from unittest.mock import patch, call from ietf.utils.test_utils import TestCase +from ietf.utils.timezone import date_today from .factories import MeetingFactory -from .tasks import proceedings_content_refresh_task, agenda_data_refresh +from .tasks import ( + proceedings_content_refresh_task, + agenda_data_refresh_task, + agenda_data_refresh_all_task, +) +from .tasks import fetch_meeting_attendance_task class TaskTests(TestCase): @patch("ietf.meeting.tasks.generate_agenda_data") - def test_agenda_data_refresh(self, mock_generate): - agenda_data_refresh() + def test_agenda_data_refresh_task(self, mock_generate): + agenda_data_refresh_task() self.assertTrue(mock_generate.called) - self.assertEqual(mock_generate.call_args, call(force_refresh=True)) + self.assertEqual(mock_generate.call_args, call(None, force_refresh=True)) + + mock_generate.reset_mock() + mock_generate.side_effect = RuntimeError + try: + agenda_data_refresh_task() + except Exception as err: + self.fail( + f"agenda_data_refresh_task should not raise exceptions (got {repr(err)})" + ) + + @patch("ietf.meeting.tasks.agenda_data_refresh_task") + @patch("ietf.meeting.tasks.chain") + def test_agenda_data_refresh_all_task(self, mock_chain, mock_agenda_data_refresh): + # Patch the agenda_data_refresh_task task with a mock whose `.map` attribute + # converts its argument, which is expected to be an iterator, to a list + # and returns it. We'll use this to check that the expected task chain + # was set up, but we don't actually run any celery tasks. + mock_agenda_data_refresh.map.side_effect = lambda x: list(x) + + meetings = MeetingFactory.create_batch(5, type_id="ietf") + numbers = sorted(int(m.number) for m in meetings) + agenda_data_refresh_all_task(batch_size=2) + self.assertTrue(mock_chain.called) + # The lists in the call() below are the output of the lambda we patched in + # via mock_agenda_data_refresh.map.side_effect above. I.e., this tests that + # map() was called with the correct batched data. + self.assertEqual( + mock_chain.call_args, + call( + [numbers[0], numbers[1]], + [numbers[2], numbers[3]], + [numbers[4]], + ), + ) + self.assertEqual(mock_agenda_data_refresh.call_count, 0) + self.assertEqual(mock_agenda_data_refresh.map.call_count, 3) @patch("ietf.meeting.tasks.generate_proceedings_content") def test_proceedings_content_refresh_task(self, mock_generate): # Generate a couple of meetings meeting120 = MeetingFactory(type_id="ietf", number="120") # 24 * 5 meeting127 = MeetingFactory(type_id="ietf", number="127") # 24 * 5 + 7 - + # Times to be returned - now_utc = datetime.datetime.now(tz=datetime.timezone.utc) + now_utc = datetime.datetime.now(tz=datetime.UTC) hour_00_utc = now_utc.replace(hour=0) hour_01_utc = now_utc.replace(hour=1) hour_07_utc = now_utc.replace(hour=7) @@ -32,20 +74,48 @@ def test_proceedings_content_refresh_task(self, mock_generate): self.assertEqual(mock_generate.call_count, 1) self.assertEqual(mock_generate.call_args, call(meeting120, force_refresh=True)) mock_generate.reset_mock() - + # hour 01 - should call no meetings with patch("ietf.meeting.tasks.timezone.now", return_value=hour_01_utc): proceedings_content_refresh_task() self.assertEqual(mock_generate.call_count, 0) - + # hour 07 - should call meeting with number % 24 == 0 with patch("ietf.meeting.tasks.timezone.now", return_value=hour_07_utc): proceedings_content_refresh_task() self.assertEqual(mock_generate.call_count, 1) self.assertEqual(mock_generate.call_args, call(meeting127, force_refresh=True)) mock_generate.reset_mock() - + # With all=True, all should be called regardless of time. Reuse hour_01_utc which called none before with patch("ietf.meeting.tasks.timezone.now", return_value=hour_01_utc): proceedings_content_refresh_task(all=True) self.assertEqual(mock_generate.call_count, 2) + + @patch("ietf.meeting.tasks.fetch_attendance_from_meetings") + def test_fetch_meeting_attendance_task(self, mock_fetch_attendance): + today = date_today() + meetings = [ + MeetingFactory(type_id="ietf", date=today - datetime.timedelta(days=1)), + MeetingFactory(type_id="ietf", date=today - datetime.timedelta(days=2)), + MeetingFactory(type_id="ietf", date=today - datetime.timedelta(days=3)), + ] + data = { + "created": 1, + "updated": 2, + "deleted": 0, + "processed": 3, + } + + mock_fetch_attendance.return_value = [data, data] + + fetch_meeting_attendance_task() + self.assertEqual(mock_fetch_attendance.call_count, 1) + self.assertCountEqual(mock_fetch_attendance.call_args[0][0], meetings[0:2]) + + # test handling of RuntimeError + mock_fetch_attendance.reset_mock() + mock_fetch_attendance.side_effect = RuntimeError + fetch_meeting_attendance_task() + self.assertTrue(mock_fetch_attendance.called) + # Good enough that we got here without raising an exception diff --git a/ietf/meeting/tests_utils.py b/ietf/meeting/tests_utils.py new file mode 100644 index 0000000000..7dd8f435e1 --- /dev/null +++ b/ietf/meeting/tests_utils.py @@ -0,0 +1,309 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +# -*- coding: utf-8 -*- + +import copy +import datetime +import debug # pyflakes: ignore +import json +import jsonschema +from json import JSONDecodeError +from unittest.mock import patch, Mock + +from django.http import HttpResponse, JsonResponse +from ietf.meeting.factories import MeetingFactory, RegistrationFactory, RegistrationTicketFactory +from ietf.meeting.models import Registration +from ietf.meeting.utils import ( + process_single_registration, + get_registration_data, + sync_registration_data, + fetch_attendance_from_meetings, + get_activity_stats +) +from ietf.nomcom.models import Volunteer +from ietf.nomcom.factories import NomComFactory, nomcom_kwargs_for_year +from ietf.person.factories import PersonFactory +from ietf.utils.test_utils import TestCase +from ietf.meeting.test_data import make_meeting_test_data +from ietf.doc.factories import NewRevisionDocEventFactory, DocEventFactory + + +class JsonResponseWithJson(JsonResponse): + def json(self): + return json.loads(self.content) + + +class ActivityStatsTests(TestCase): + + def test_activity_stats(self): + utc = datetime.timezone.utc + make_meeting_test_data() + sdate = datetime.date(2016,4,3) + edate = datetime.date(2016,7,14) + MeetingFactory(type_id='ietf', date=sdate, number="96") + MeetingFactory(type_id='ietf', date=edate, number="97") + + NewRevisionDocEventFactory(time=datetime.datetime(2016,4,5,12,0,0,0,tzinfo=utc)) + NewRevisionDocEventFactory(time=datetime.datetime(2016,4,6,12,0,0,0,tzinfo=utc)) + NewRevisionDocEventFactory(time=datetime.datetime(2016,4,7,12,0,0,0,tzinfo=utc)) + + NewRevisionDocEventFactory(time=datetime.datetime(2016,6,30,12,0,0,0,tzinfo=utc)) + NewRevisionDocEventFactory(time=datetime.datetime(2016,6,30,13,0,0,0,tzinfo=utc)) + + DocEventFactory(doc__std_level_id="ps", doc__type_id="rfc", type="published_rfc", time=datetime.datetime(2016,4,5,12,0,0,0,tzinfo=utc)) + DocEventFactory(doc__std_level_id="bcp", doc__type_id="rfc", type="published_rfc", time=datetime.datetime(2016,4,6,12,0,0,0,tzinfo=utc)) + DocEventFactory(doc__std_level_id="inf", doc__type_id="rfc", type="published_rfc", time=datetime.datetime(2016,4,7,12,0,0,0,tzinfo=utc)) + DocEventFactory(doc__std_level_id="exp", doc__type_id="rfc", type="published_rfc", time=datetime.datetime(2016,4,8,12,0,0,0,tzinfo=utc)) + + data = get_activity_stats(sdate, edate) + self.assertEqual(data['new_drafts_count'], len(data['new_docs'])) + self.assertEqual(data['ffw_new_count'], 2) + self.assertEqual(data['ffw_new_percent'], '40%') + rfc_count = 0 + for c in data['counts']: + rfc_count += data['counts'].get(c) + self.assertEqual(rfc_count, len(data['rfcs'])) + + +class GetRegistrationsTests(TestCase): + + @patch('ietf.meeting.utils.requests.get') + def test_get_registation_data(self, mock_get): + meeting = MeetingFactory(type_id='ietf', number='122') + person = PersonFactory() + reg_details = dict( + first_name=person.first_name(), + last_name=person.last_name(), + email=person.email().address, + affiliation='Microsoft', + country_code='US', + meeting=meeting.number, + checkedin=True, + is_nomcom_volunteer=False, + cancelled=False, + tickets=[{'attendance_type': 'onsite', 'ticket_type': 'week_pass'}], + ) + reg_data = {'objects': {person.email().address: reg_details}} + reg_data_bad = copy.deepcopy(reg_data) + del reg_data_bad['objects'][person.email().address]['email'] + response1 = HttpResponse('Invalid apikey', status=403) + response2 = JsonResponseWithJson(reg_data) + response3 = Mock() + response3.status_code = 200 + response3.json.side_effect = JSONDecodeError("Expecting value", doc="", pos=0) + response4 = JsonResponseWithJson(reg_data_bad) + mock_get.side_effect = [response1, response2, response3, response4] + # test status 403 + with self.assertRaises(Exception): + get_registration_data(meeting) + # test status 200 good + returned_data = get_registration_data(meeting) + self.assertEqual(returned_data, reg_data) + # test decode error + with self.assertRaises(ValueError): + get_registration_data(meeting) + # test validation error + with self.assertRaises(jsonschema.exceptions.ValidationError): + get_registration_data(meeting) + + @patch('ietf.meeting.utils.get_registration_data') + def test_sync_registation_data(self, mock_get): + meeting = MeetingFactory(type_id='ietf', number='122') + person1 = PersonFactory() + person2 = PersonFactory() + items = [] + for person in [person1, person2]: + items.append(dict( + first_name=person.first_name(), + last_name=person.last_name(), + email=person.email().address, + affiliation='Microsoft', + country_code='US', + meeting=meeting.number, + checkedin=True, + is_nomcom_volunteer=False, + cancelled=False, + tickets=[{'attendance_type': 'onsite', 'ticket_type': 'week_pass'}], + )) + reg_data = {'objects': {items[0]['email']: items[0], items[1]['email']: items[1]}} + mock_get.return_value = reg_data + self.assertEqual(Registration.objects.filter(meeting=meeting).count(), 0) + stats = sync_registration_data(meeting) + self.assertEqual(Registration.objects.filter(meeting=meeting).count(), 2) + self.assertEqual(stats['created'], 2) + # test idempotent + stats = sync_registration_data(meeting) + self.assertEqual(Registration.objects.filter(meeting=meeting).count(), 2) + self.assertEqual(stats['created'], 0) + # test delete cancelled registration + del reg_data['objects'][items[1]['email']] + stats = sync_registration_data(meeting) + self.assertEqual(Registration.objects.filter(meeting=meeting).count(), 1) + self.assertEqual(stats['deleted'], 1) + + def test_process_single_registration(self): + # test new registration + meeting = MeetingFactory(type_id='ietf', number='122') + person = PersonFactory() + reg_data = dict( + first_name=person.first_name(), + last_name=person.last_name(), + email=person.email().address, + affiliation='Microsoft', + country_code='US', + meeting=meeting.number, + checkedin=True, + is_nomcom_volunteer=False, + cancelled=False, + tickets=[{'attendance_type': 'onsite', 'ticket_type': 'week_pass'}], + ) + self.assertEqual(meeting.registration_set.count(), 0) + new_reg, action = process_single_registration(reg_data, meeting) + self.assertEqual(meeting.registration_set.count(), 1) + reg = meeting.registration_set.first() + self.assertEqual(new_reg, reg) + self.assertEqual(action, 'created') + self.assertEqual(reg.first_name, person.first_name()) + self.assertEqual(reg.last_name, person.last_name()) + self.assertEqual(reg.email, person.email().address) + self.assertEqual(reg.affiliation, 'Microsoft') + self.assertEqual(reg.meeting, meeting) + self.assertEqual(reg.checkedin, True) + self.assertEqual(reg.tickets.count(), 1) + ticket = reg.tickets.first() + self.assertEqual(ticket.attendance_type.slug, 'onsite') + self.assertEqual(ticket.ticket_type.slug, 'week_pass') + + # test no change + new_reg, action = process_single_registration(reg_data, meeting) + self.assertEqual(meeting.registration_set.count(), 1) + reg = meeting.registration_set.first() + self.assertEqual(new_reg, reg) + self.assertEqual(action, None) + + # test update fields + reg_data['affiliation'] = 'Cisco' + new_reg, action = process_single_registration(reg_data, meeting) + self.assertEqual(meeting.registration_set.count(), 1) + reg = meeting.registration_set.first() + self.assertEqual(new_reg, reg) + self.assertEqual(action, 'updated') + self.assertEqual(reg.affiliation, 'Cisco') + + # test update tickets + reg_data['tickets'] = [{'attendance_type': 'remote', 'ticket_type': 'week_pass'}] + new_reg, action = process_single_registration(reg_data, meeting) + self.assertEqual(meeting.registration_set.count(), 1) + reg = meeting.registration_set.first() + self.assertEqual(new_reg, reg) + self.assertEqual(action, 'updated') + self.assertEqual(reg.tickets.count(), 1) + ticket = reg.tickets.first() + self.assertEqual(ticket.attendance_type.slug, 'remote') + + # test tickets, two of same + reg_data['tickets'] = [ + {'attendance_type': 'onsite', 'ticket_type': 'one_day'}, + {'attendance_type': 'onsite', 'ticket_type': 'one_day'}, + {'attendance_type': 'remote', 'ticket_type': 'week_pass'}, + ] + new_reg, action = process_single_registration(reg_data, meeting) + self.assertEqual(meeting.registration_set.count(), 1) + reg = meeting.registration_set.first() + self.assertEqual(new_reg, reg) + self.assertEqual(action, 'updated') + self.assertEqual(reg.tickets.count(), 3) + self.assertEqual(reg.tickets.filter(attendance_type__slug='onsite', ticket_type__slug='one_day').count(), 2) + self.assertEqual(reg.tickets.filter(attendance_type__slug='remote', ticket_type__slug='week_pass').count(), 1) + + # test tickets, two of same, delete one + reg_data['tickets'] = [ + {'attendance_type': 'onsite', 'ticket_type': 'one_day'}, + {'attendance_type': 'remote', 'ticket_type': 'week_pass'}, + ] + new_reg, action = process_single_registration(reg_data, meeting) + self.assertEqual(meeting.registration_set.count(), 1) + reg = meeting.registration_set.first() + self.assertEqual(new_reg, reg) + self.assertEqual(action, 'updated') + self.assertEqual(reg.tickets.count(), 2) + self.assertEqual(reg.tickets.filter(attendance_type__slug='onsite', ticket_type__slug='one_day').count(), 1) + self.assertEqual(reg.tickets.filter(attendance_type__slug='remote', ticket_type__slug='week_pass').count(), 1) + + def test_process_single_registration_nomcom(self): + '''Test that Volunteer is created if is_nomcom_volunteer=True''' + meeting = MeetingFactory(type_id='ietf', number='122') + person = PersonFactory() + reg_data = dict( + first_name=person.first_name(), + last_name=person.last_name(), + email=person.email().address, + affiliation='Microsoft', + country_code='US', + meeting=meeting.number, + checkedin=True, + is_nomcom_volunteer=True, + cancelled=False, + tickets=[{'attendance_type': 'onsite', 'ticket_type': 'week_pass'}], + ) + now = datetime.datetime.now() + if now.month > 10: + year = now.year + 1 + else: + year = now.year + # create appropriate group and nomcom objects + nomcom = NomComFactory.create(is_accepting_volunteers=True, **nomcom_kwargs_for_year(year)) + # assert no Volunteers exists + self.assertEqual(Volunteer.objects.count(), 0) + new_reg, action = process_single_registration(reg_data, meeting) + self.assertEqual(action, 'created') + # assert Volunteer exists + self.assertEqual(Volunteer.objects.count(), 1) + volunteer = Volunteer.objects.last() + self.assertEqual(volunteer.person, person) + self.assertEqual(volunteer.nomcom, nomcom) + self.assertEqual(volunteer.origin, 'registration') + + def test_process_single_registration_cancelled(self): + # test cancelled registration, one of two tickets + meeting = MeetingFactory(type_id='ietf', number='122') + person = PersonFactory() + reg = RegistrationFactory(meeting=meeting, person=person, checkedin=False, with_ticket={'attendance_type_id': 'onsite'}) + RegistrationTicketFactory(registration=reg, attendance_type_id='remote', ticket_type_id='week_pass') + reg_data = dict( + first_name=person.first_name(), + last_name=person.last_name(), + email=person.email().address, + affiliation='Microsoft', + country_code='US', + meeting=meeting.number, + checkedin=False, + is_nomcom_volunteer=False, + cancelled=True, + tickets=[{'attendance_type': 'onsite', 'ticket_type': 'week_pass'}], + ) + self.assertEqual(meeting.registration_set.count(), 1) + self.assertEqual(reg.tickets.count(), 2) + new_reg, action = process_single_registration(reg_data, meeting) + self.assertEqual((new_reg, action), (None, 'deleted')) + self.assertEqual(meeting.registration_set.count(), 1) + self.assertEqual(reg.tickets.count(), 1) + self.assertTrue(reg.tickets.filter(attendance_type__slug='remote').exists()) + # test cancelled registration, last ticket + reg_data['tickets'][0]['attendance_type'] = 'remote' + new_reg, action = process_single_registration(reg_data, meeting) + self.assertEqual((new_reg, action), (None, 'deleted')) + self.assertEqual(meeting.registration_set.count(), 0) + + @patch("ietf.meeting.utils.sync_registration_data") + def test_fetch_attendance_from_meetings(self, mock_sync_reg_data): + mock_meetings = [object(), object(), object()] + d1 = dict(created=1, updated=2, deleted=0, processed=3) + d2 = dict(created=2, updated=2, deleted=0, processed=4) + d3 = dict(created=1, updated=4, deleted=1, processed=5) + mock_sync_reg_data.side_effect = (d1, d2, d3) + stats = fetch_attendance_from_meetings(mock_meetings) + self.assertEqual( + [mock_sync_reg_data.call_args_list[n][0][0] for n in range(3)], + mock_meetings, + ) + self.assertEqual(stats, [d1, d2, d3]) diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index dfb414b61b..17988e50be 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2009-2024, All Rights Reserved +# Copyright The IETF Trust 2009-2025, All Rights Reserved # -*- coding: utf-8 -*- import datetime import io @@ -12,9 +12,10 @@ import requests_mock from unittest import skipIf -from mock import call, patch, PropertyMock +from unittest.mock import call, patch, PropertyMock from pyquery import PyQuery from lxml.etree import tostring +from icalendar import Calendar from io import StringIO, BytesIO from bs4 import BeautifulSoup from urllib.parse import urlparse, urlsplit @@ -32,6 +33,7 @@ from django.http import QueryDict, FileResponse from django.template import Context, Template from django.utils import timezone +from django.utils.html import escape from django.utils.safestring import mark_safe from django.utils.text import slugify @@ -48,15 +50,18 @@ from ietf.meeting.helpers import send_interim_minutes_reminder, populate_important_dates, update_important_dates from ietf.meeting.models import Session, TimeSlot, Meeting, SchedTimeSessAssignment, Schedule, SessionPresentation, SlideSubmission, SchedulingEvent, Room, Constraint, ConstraintName from ietf.meeting.test_data import make_meeting_test_data, make_interim_meeting, make_interim_test_data -from ietf.meeting.utils import condition_slide_order, generate_proceedings_content +from ietf.meeting.utils import ( + condition_slide_order, + generate_proceedings_content, + diff_meeting_schedules, +) from ietf.meeting.utils import add_event_info_to_session_qs, participants_for_meeting from ietf.meeting.utils import create_recording, delete_recording, get_next_sequence, bluesheet_data from ietf.meeting.views import session_draft_list, parse_agenda_filter_params, sessions_post_save, agenda_extract_schedule from ietf.meeting.views import get_summary_by_area, get_summary_by_type, get_summary_by_purpose, generate_agenda_data from ietf.name.models import SessionStatusName, ImportantDateName, RoleName, ProceedingsMaterialTypeName -from ietf.utils.decorators import skip_coverage from ietf.utils.mail import outbox, empty_outbox, get_payload_text -from ietf.utils.test_runner import TestBlobstoreManager +from ietf.utils.test_runner import TestBlobstoreManager, disable_coverage from ietf.utils.test_utils import TestCase, login_testing_unauthorized, unicontent from ietf.utils.timezone import date_today, time_now @@ -66,8 +71,7 @@ SessionPresentationFactory, MeetingFactory, FloorPlanFactory, TimeSlotFactory, SlideSubmissionFactory, RoomFactory, ConstraintFactory, MeetingHostFactory, ProceedingsMaterialFactory, - AttendedFactory) -from ietf.stats.factories import MeetingRegistrationFactory + AttendedFactory, RegistrationFactory) from ietf.doc.factories import DocumentFactory, WgDraftFactory from ietf.submit.tests import submission_file from ietf.utils.test_utils import assert_ical_response_is_valid @@ -321,11 +325,11 @@ def test_meeting_agenda(self): self.assertContains(r, session.group.parent.acronym.upper()) self.assertContains(r, slot.location.name) self.assertContains(r, "{}-{}".format( - slot.time.astimezone(datetime.timezone.utc).strftime("%H%M"), - (slot.time + slot.duration).astimezone(datetime.timezone.utc).strftime("%H%M"), + slot.time.astimezone(datetime.UTC).strftime("%H%M"), + (slot.time + slot.duration).astimezone(datetime.UTC).strftime("%H%M"), )) self.assertContains(r, "shown in UTC") - updated = meeting.updated().astimezone(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S %Z") + updated = meeting.updated().astimezone(datetime.UTC).strftime("%Y-%m-%d %H:%M:%S %Z") self.assertContains(r, f"Updated {updated}") # text, invalid updated (none) @@ -369,8 +373,8 @@ def test_meeting_agenda(self): self.assertContains(r, session.group.parent.acronym.upper()) self.assertContains(r, slot.location.name) self.assertContains(r, registration_text) - start_time = slot.time.astimezone(datetime.timezone.utc) - end_time = slot.end_time().astimezone(datetime.timezone.utc) + start_time = slot.time.astimezone(datetime.UTC) + end_time = slot.end_time().astimezone(datetime.UTC) self.assertContains(r, '"{}","{}","{}"'.format( start_time.strftime("%Y-%m-%d"), start_time.strftime("%H%M"), @@ -385,9 +389,6 @@ def test_meeting_agenda(self): r = self.client.get(ical_url) assert_ical_response_is_valid(self, r) - self.assertContains(r, "BEGIN:VTIMEZONE") - self.assertContains(r, "END:VTIMEZONE") - self.assertContains(r, meeting.time_zone, msg_prefix="time_zone should appear in its original case") self.assertNotEqual( meeting.time_zone, meeting.time_zone.lower(), @@ -406,21 +407,32 @@ def test_meeting_agenda(self): assert_ical_response_is_valid(self, r) self.assertContains(r, session.group.acronym) self.assertContains(r, session.group.name) - self.assertContains(r, session.remote_instructions) - self.assertContains(r, slot.location.name) - self.assertContains(r, 'https://onsite.example.com') - self.assertContains(r, 'https://meetecho.example.com') - self.assertContains(r, "BEGIN:VTIMEZONE") - self.assertContains(r, "END:VTIMEZONE") - self.assertContains(r, session.agenda().get_href()) - self.assertContains( - r, + cal = Calendar.from_ical(r.content) + events = [component for component in cal.walk() if component.name == "VEVENT"] + + self.assertEqual(len(events), 2) + self.assertIn(session.remote_instructions, events[0].get('description')) + self.assertIn("Onsite tool: https://onsite.example.com", events[0].get('description')) + self.assertIn("Meetecho: https://meetecho.example.com", events[0].get('description')) + self.assertIn(f"Agenda {session.agenda().get_href()}", events[0].get('description')) + session_materials_url = settings.IDTRACKER_BASE_URL + urlreverse( + 'ietf.meeting.views.session_details', + kwargs=dict(num=meeting.number, acronym=session.group.acronym) + ) + self.assertIn(f"Session materials: {session_materials_url}", events[0].get('description')) + self.assertIn( urlreverse( 'ietf.meeting.views.session_details', kwargs=dict(num=meeting.number, acronym=session.group.acronym)), - msg_prefix='ical should contain link to meeting materials page for session') + events[0].get('description')) + self.assertEqual( + session_materials_url, + events[0].get('url') + ) + self.assertContains(r, f"LOCATION:{slot.location.name}") + # Floor Plan r = self.client.get(urlreverse('floor-plan', kwargs=dict(num=meeting.number))) self.assertEqual(r.status_code, 200) @@ -1029,7 +1041,7 @@ def test_important_dates_ical(self): updated = meeting.updated() self.assertIsNotNone(updated) - expected_updated = updated.astimezone(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + expected_updated = updated.astimezone(datetime.UTC).strftime("%Y%m%dT%H%M%SZ") self.assertContains(r, f"DTSTAMP:{expected_updated}") dtstamps_count = r.content.decode("utf-8").count(f"DTSTAMP:{expected_updated}") self.assertEqual(dtstamps_count, meeting.importantdate_set.count()) @@ -1050,32 +1062,36 @@ def test_group_ical(self): s1 = Session.objects.filter(meeting=meeting, group__acronym="mars").first() a1 = s1.official_timeslotassignment() t1 = a1.timeslot + # Create an extra session t2 = TimeSlotFactory.create( meeting=meeting, - time=meeting.tz().localize( + time=pytz.utc.localize( datetime.datetime.combine(meeting.date, datetime.time(11, 30)) ) ) + s2 = SessionFactory.create(meeting=meeting, group=s1.group, add_to_schedule=False) SchedTimeSessAssignment.objects.create(timeslot=t2, session=s2, schedule=meeting.schedule) - # + url = urlreverse('ietf.meeting.views.agenda_ical', kwargs={'num':meeting.number, 'acronym':s1.group.acronym, }) r = self.client.get(url) assert_ical_response_is_valid(self, r, expected_event_summaries=['mars - Martian Special Interest Group'], expected_event_count=2) - self.assertContains(r, t1.local_start_time().strftime('%Y%m%dT%H%M%S')) - self.assertContains(r, t2.local_start_time().strftime('%Y%m%dT%H%M%S')) - # + self.assertContains(r, f"DTSTART:{t1.time.strftime('%Y%m%dT%H%M%SZ')}") + self.assertContains(r, f"DTEND:{(t1.time + t1.duration).strftime('%Y%m%dT%H%M%SZ')}") + self.assertContains(r, f"DTSTART:{t2.time.strftime('%Y%m%dT%H%M%SZ')}") + self.assertContains(r, f"DTEND:{(t2.time + t2.duration).strftime('%Y%m%dT%H%M%SZ')}") + url = urlreverse('ietf.meeting.views.agenda_ical', kwargs={'num':meeting.number, 'session_id':s1.id, }) r = self.client.get(url) assert_ical_response_is_valid(self, r, expected_event_summaries=['mars - Martian Special Interest Group'], expected_event_count=1) - self.assertContains(r, t1.local_start_time().strftime('%Y%m%dT%H%M%S')) - self.assertNotContains(r, t2.local_start_time().strftime('%Y%m%dT%H%M%S')) + self.assertContains(r, f"DTSTART:{t1.time.strftime('%Y%m%dT%H%M%SZ')}") + self.assertNotContains(r, f"DTSTART:{t2.time.strftime('%Y%m%dT%H%M%SZ')}") def test_parse_agenda_filter_params(self): def _r(show=(), hide=(), showtypes=(), hidetypes=()): @@ -1169,8 +1185,8 @@ def test_session_draft_tarfile(self): os.unlink(filename) @skipIf(skip_pdf_tests, skip_message) - @skip_coverage - def test_session_draft_pdf(self): + @disable_coverage() + def test_session_draft_pdf(self): # pragma: no cover session, filenames = self.build_session_setup() try: url = urlreverse('ietf.meeting.views.session_draft_pdf', kwargs={'num':session.meeting.number,'acronym':session.group.acronym}) @@ -2105,8 +2121,8 @@ def test_editor_time_zone(self): # strftime() does not seem to support hours without leading 0, so do this manually time_label_string = f'{ts_start.hour:d}:{ts_start.minute:02d} - {ts_end.hour:d}:{ts_end.minute:02d}' self.assertIn(time_label_string, time_label.text()) - self.assertEqual(time_label.attr('data-start'), ts_start.astimezone(datetime.timezone.utc).isoformat()) - self.assertEqual(time_label.attr('data-end'), ts_end.astimezone(datetime.timezone.utc).isoformat()) + self.assertEqual(time_label.attr('data-start'), ts_start.astimezone(datetime.UTC).isoformat()) + self.assertEqual(time_label.attr('data-end'), ts_end.astimezone(datetime.UTC).isoformat()) ts_swap = time_label.find('.swap-timeslot-col') origin_label = ts_swap.attr('data-origin-label') @@ -2117,8 +2133,8 @@ def test_editor_time_zone(self): timeslot_elt = pq(f'#timeslot{timeslot.pk}') self.assertEqual(len(timeslot_elt), 1) - self.assertEqual(timeslot_elt.attr('data-start'), ts_start.astimezone(datetime.timezone.utc).isoformat()) - self.assertEqual(timeslot_elt.attr('data-end'), ts_end.astimezone(datetime.timezone.utc).isoformat()) + self.assertEqual(timeslot_elt.attr('data-start'), ts_start.astimezone(datetime.UTC).isoformat()) + self.assertEqual(timeslot_elt.attr('data-end'), ts_end.astimezone(datetime.UTC).isoformat()) timeslot_label = pq(f'#timeslot{timeslot.pk} .time-label') self.assertEqual(len(timeslot_label), 1) @@ -4739,7 +4755,7 @@ def _approval_url(slidesub): 0, "second session proposed slides should be linked for approval", ) - + class EditScheduleListTests(TestCase): def setUp(self): @@ -4754,73 +4770,151 @@ def test_list_schedules(self): self.assertTrue(r.status_code, 200) def test_diff_schedules(self): - meeting = make_meeting_test_data() - - url = urlreverse('ietf.meeting.views.diff_schedules',kwargs={'num':meeting.number}) - login_testing_unauthorized(self,"secretary", url) - r = self.client.get(url) - self.assertTrue(r.status_code, 200) - - from_schedule = Schedule.objects.get(meeting=meeting, name="test-unofficial-schedule") - - session1 = Session.objects.filter(meeting=meeting, group__acronym='mars').first() - session2 = Session.objects.filter(meeting=meeting, group__acronym='ames').first() - session3 = SessionFactory(meeting=meeting, group=Group.objects.get(acronym='mars'), - attendees=10, requested_duration=datetime.timedelta(minutes=70), - add_to_schedule=False) - SchedulingEvent.objects.create(session=session3, status_id='schedw', by=Person.objects.first()) - - slot2 = TimeSlot.objects.filter(meeting=meeting, type='regular').order_by('-time').first() - slot3 = TimeSlot.objects.create( - meeting=meeting, type_id='regular', location=slot2.location, - duration=datetime.timedelta(minutes=60), - time=slot2.time + datetime.timedelta(minutes=60), + # Create meeting and some time slots + meeting = MeetingFactory(type_id="ietf", populate_schedule=False) + rooms = RoomFactory.create_batch(2, meeting=meeting) + # first index is room, second is time + timeslots = [ + [ + TimeSlotFactory( + location=room, + meeting=meeting, + time=datetime.datetime.combine( + meeting.date, datetime.time(9, 0, tzinfo=datetime.UTC) + ) + ), + TimeSlotFactory( + location=room, + meeting=meeting, + time=datetime.datetime.combine( + meeting.date, datetime.time(10, 0, tzinfo=datetime.UTC) + ) + ), + TimeSlotFactory( + location=room, + meeting=meeting, + time=datetime.datetime.combine( + meeting.date, datetime.time(11, 0, tzinfo=datetime.UTC) + ) + ), + ] + for room in rooms + ] + sessions = SessionFactory.create_batch( + 5, meeting=meeting, add_to_schedule=False ) - # copy - new_url = urlreverse("ietf.meeting.views.new_meeting_schedule", kwargs=dict(num=meeting.number, owner=from_schedule.owner_email(), name=from_schedule.name)) - r = self.client.post(new_url, { - 'name': "newtest", - 'public': "on", - }) - self.assertNoFormPostErrors(r) + from_schedule = ScheduleFactory(meeting=meeting) + to_schedule = ScheduleFactory(meeting=meeting) - to_schedule = Schedule.objects.get(meeting=meeting, name='newtest') + # sessions[0]: not scheduled in from_schedule, scheduled in to_schedule + SchedTimeSessAssignment.objects.create( + schedule=to_schedule, + session=sessions[0], + timeslot=timeslots[0][0], + ) + # sessions[1]: scheduled in from_schedule, not scheduled in to_schedule + SchedTimeSessAssignment.objects.create( + schedule=from_schedule, + session=sessions[1], + timeslot=timeslots[0][0], + ) + # sessions[2]: moves rooms, not time + SchedTimeSessAssignment.objects.create( + schedule=from_schedule, + session=sessions[2], + timeslot=timeslots[0][1], + ) + SchedTimeSessAssignment.objects.create( + schedule=to_schedule, + session=sessions[2], + timeslot=timeslots[1][1], + ) + # sessions[3]: moves time, not room + SchedTimeSessAssignment.objects.create( + schedule=from_schedule, + session=sessions[3], + timeslot=timeslots[1][1], + ) + SchedTimeSessAssignment.objects.create( + schedule=to_schedule, + session=sessions[3], + timeslot=timeslots[1][2], + ) + # sessions[4]: moves room and time + SchedTimeSessAssignment.objects.create( + schedule=from_schedule, + session=sessions[4], + timeslot=timeslots[1][0], + ) + SchedTimeSessAssignment.objects.create( + schedule=to_schedule, + session=sessions[4], + timeslot=timeslots[0][2], + ) - # make some changes + # Check the raw diffs + raw_diffs = diff_meeting_schedules(from_schedule, to_schedule) + self.assertCountEqual( + raw_diffs, + [ + { + "change": "schedule", + "session": sessions[0].pk, + "to": timeslots[0][0].pk, + }, + { + "change": "unschedule", + "session": sessions[1].pk, + "from": timeslots[0][0].pk, + }, + { + "change": "move", + "session": sessions[2].pk, + "from": timeslots[0][1].pk, + "to": timeslots[1][1].pk, + }, + { + "change": "move", + "session": sessions[3].pk, + "from": timeslots[1][1].pk, + "to": timeslots[1][2].pk, + }, + { + "change": "move", + "session": sessions[4].pk, + "from": timeslots[1][0].pk, + "to": timeslots[0][2].pk, + }, + ] + ) - edit_url = urlreverse("ietf.meeting.views.edit_meeting_schedule", kwargs=dict(num=meeting.number, owner=to_schedule.owner_email(), name=to_schedule.name)) + # Check the view + url = urlreverse("ietf.meeting.views.diff_schedules", + kwargs={"num": meeting.number}) + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + self.assertTrue(r.status_code, 200) - # schedule session - r = self.client.post(edit_url, { - 'action': 'assign', - 'timeslot': slot3.pk, - 'session': session3.pk, - }) - self.assertEqual(json.loads(r.content)['success'], True) - # unschedule session - r = self.client.post(edit_url, { - 'action': 'unassign', - 'session': session1.pk, - }) - self.assertEqual(json.loads(r.content)['success'], True) - # move session - r = self.client.post(edit_url, { - 'action': 'assign', - 'timeslot': slot2.pk, - 'session': session2.pk, + # with show room changes disabled - does not show sessions[2] because it did + # not change time + r = self.client.get(url, { + "from_schedule": from_schedule.name, + "to_schedule": to_schedule.name, }) - self.assertEqual(json.loads(r.content)['success'], True) + self.assertTrue(r.status_code, 200) + q = PyQuery(r.content) + self.assertEqual(len(q(".schedule-diffs tr")), 4 + 1) - # now get differences + # with show room changes enabled - shows all changes r = self.client.get(url, { - 'from_schedule': from_schedule.name, - 'to_schedule': to_schedule.name, + "from_schedule": from_schedule.name, + "to_schedule": to_schedule.name, + "show_room_changes": "on", }) self.assertTrue(r.status_code, 200) - q = PyQuery(r.content) - self.assertEqual(len(q(".schedule-diffs tr")), 3+1) + self.assertEqual(len(q(".schedule-diffs tr")), 5 + 1) def test_delete_schedule(self): url = urlreverse('ietf.meeting.views.delete_schedule', @@ -5209,7 +5303,9 @@ def test_upcoming_ical(self): assert_ical_response_is_valid(self, r, expected_event_summaries=expected_event_summaries, expected_event_count=len(expected_event_summaries)) - self.assertContains(r, 'Remote instructions: https://someurl.example.com') + # Unfold long lines that might have been folded by iCal + content_unfolded = r.content.decode('utf-8').replace('\r\n ', '') + self.assertIn('Remote instructions: https://someurl.example.com', content_unfolded) Session.objects.filter(meeting__type_id='interim').update(remote_instructions='') r = self.client.get(url) @@ -5217,11 +5313,12 @@ def test_upcoming_ical(self): assert_ical_response_is_valid(self, r, expected_event_summaries=expected_event_summaries, expected_event_count=len(expected_event_summaries)) - self.assertNotContains(r, 'Remote instructions:') + content_unfolded = r.content.decode('utf-8').replace('\r\n ', '') + self.assertNotIn('Remote instructions:', content_unfolded) updated = meeting.updated() self.assertIsNotNone(updated) - expected_updated = updated.astimezone(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + expected_updated = updated.astimezone(datetime.UTC).strftime("%Y%m%dT%H%M%SZ") self.assertContains(r, f"DTSTAMP:{expected_updated}") # With default cached_updated, 1970-01-01 @@ -7055,7 +7152,7 @@ def test_disapprove_proposed_slides(self): self.assertFalse(exists_in_storage("staging", submission.filename)) r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertRegex(r.content.decode(), r"These\s+slides\s+have\s+already\s+been\s+rejected") + self.assertRegex(r.content.decode(), r"These\s+slides\s+have\s+already\s+been\s+declined") @override_settings(MEETECHO_API_CONFIG="fake settings") # enough to trigger API calls @patch("ietf.meeting.views.SlidesManager") @@ -7249,6 +7346,67 @@ def test_submit_and_approve_multiple_versions(self, mock_slides_manager_cls): fd.close() self.assertIn('third version', contents) + @override_settings( + MEETECHO_API_CONFIG="fake settings" + ) # enough to trigger API calls + @patch("ietf.meeting.views.SlidesManager") + def test_notify_meetecho_of_all_slides(self, mock_slides_manager_cls): + for meeting_type in ["ietf", "interim"]: + # Reset for the sake of the second iteration + self.client.logout() + mock_slides_manager_cls.reset_mock() + + session = SessionFactory(meeting__type_id=meeting_type) + meeting = session.meeting + + # bad meeting + url = urlreverse( + "ietf.meeting.views.notify_meetecho_of_all_slides", + kwargs={"num": 9999, "acronym": session.group.acronym}, + ) + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + self.assertEqual(r.status_code, 404) + r = self.client.post(url) + self.assertEqual(r.status_code, 404) + self.assertFalse(mock_slides_manager_cls.called) + self.client.logout() + + # good meeting + url = urlreverse( + "ietf.meeting.views.notify_meetecho_of_all_slides", + kwargs={"num": meeting.number, "acronym": session.group.acronym}, + ) + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + self.assertEqual(r.status_code, 405) + self.assertFalse(mock_slides_manager_cls.called) + mock_slides_manager = mock_slides_manager_cls.return_value + mock_slides_manager.send_update.return_value = True + r = self.client.post(url) + self.assertEqual(r.status_code, 302) + self.assertEqual(mock_slides_manager.send_update.call_count, 1) + self.assertEqual(mock_slides_manager.send_update.call_args, call(session)) + r = self.client.get(r["Location"]) + messages = list(r.context["messages"]) + self.assertEqual(len(messages), 1) + self.assertEqual( + str(messages[0]), f"Notified Meetecho about slides for {session}" + ) + + mock_slides_manager.send_update.reset_mock() + mock_slides_manager.send_update.return_value = False + r = self.client.post(url) + self.assertEqual(r.status_code, 302) + self.assertEqual(mock_slides_manager.send_update.call_count, 1) + self.assertEqual(mock_slides_manager.send_update.call_args, call(session)) + r = self.client.get(r["Location"]) + messages = list(r.context["messages"]) + self.assertEqual(len(messages), 1) + self.assertIn( + "No sessions were eligible for Meetecho slides update.", str(messages[0]) + ) + @override_settings(IETF_NOTES_URL='https://notes.ietf.org/') class ImportNotesTests(TestCase): @@ -7543,7 +7701,7 @@ def test_meeting_requests(self): ) def _sreq_edit_link(sess): return urlreverse( - 'ietf.secr.sreq.views.edit', + 'ietf.meeting.views_session_request.edit_request', kwargs={ 'num': meeting.number, 'acronym': sess.group.acronym, @@ -8679,7 +8837,8 @@ def test_generate_proceedings_content(self, mock_caches): self.assertTrue(mock_default_cache.get.called) self.assertEqual(mock_default_cache.get.call_args.args[0], cache_key, "same cache key each time") self.assertTrue(mock_default_cache.set.called) - self.assertEqual(mock_default_cache.set.call_args, call(cache_key, proceedings_content, timeout=86400)) + self.assertEqual(mock_default_cache.set.call_args.args, (cache_key, proceedings_content)) + self.assertGreater(mock_default_cache.set.call_args.kwargs["timeout"], 86400) mock_default_cache.get.reset_mock() mock_default_cache.set.reset_mock() @@ -8725,7 +8884,8 @@ def test_generate_proceedings_content(self, mock_caches): self.assertEqual(result, proceedings_content) # should have recomputed the same thing self.assertFalse(mock_default_cache.get.called, "don't bother reading cache when force_refresh is True") self.assertTrue(mock_default_cache.set.called) - self.assertEqual(mock_default_cache.set.call_args, call(cache_key, proceedings_content, timeout=86400)) + self.assertEqual(mock_default_cache.set.call_args.args, (cache_key, proceedings_content)) + self.assertGreater(mock_default_cache.set.call_args.kwargs["timeout"], 86400) def test_named_session(self): """Session with a name should appear separately in the proceedings""" @@ -8848,27 +9008,39 @@ def test_proceedings_attendees(self): - assert onsite checkedin=True appears, not onsite checkedin=False - assert remote attended appears, not remote not attended - prefer onsite checkedin=True to remote attended when same person has both + - summary stats row shows correct counts + - chart data JSON is embedded with correct values """ - meeting = MeetingFactory(type_id='ietf', date=datetime.date(2023, 11, 4), number="118") + m = MeetingFactory(type_id='ietf', date=datetime.date(2023, 11, 4), number="118") person_a = PersonFactory(name='Person A') person_b = PersonFactory(name='Person B') person_c = PersonFactory(name='Person C') person_d = PersonFactory(name='Person D') - MeetingRegistrationFactory(meeting=meeting, person=person_a, reg_type='onsite', checkedin=True) - MeetingRegistrationFactory(meeting=meeting, person=person_b, reg_type='onsite', checkedin=False) - MeetingRegistrationFactory(meeting=meeting, person=person_a, reg_type='remote') - AttendedFactory(session__meeting=meeting, session__type_id='plenary', person=person_a) - MeetingRegistrationFactory(meeting=meeting, person=person_c, reg_type='remote') - AttendedFactory(session__meeting=meeting, session__type_id='plenary', person=person_c) - MeetingRegistrationFactory(meeting=meeting, person=person_d, reg_type='remote') + areg = RegistrationFactory(meeting=m, person=person_a, checkedin=True, with_ticket={'attendance_type_id': 'onsite'}) + RegistrationFactory(meeting=m, person=person_b, checkedin=False, with_ticket={'attendance_type_id': 'onsite'}) + creg = RegistrationFactory(meeting=m, person=person_c, with_ticket={'attendance_type_id': 'remote'}) + RegistrationFactory(meeting=m, person=person_d, with_ticket={'attendance_type_id': 'remote'}) + AttendedFactory(session__meeting=m, session__type_id='plenary', person=person_a) + AttendedFactory(session__meeting=m, session__type_id='plenary', person=person_c) url = urlreverse('ietf.meeting.views.proceedings_attendees',kwargs={'num': 118}) response = self.client.get(url) self.assertContains(response, 'Attendee list') q = PyQuery(response.content) self.assertEqual(2, len(q("#id_attendees tbody tr"))) text = q('#id_attendees tbody tr').text().replace('\n', ' ') - self.assertEqual(text, "A Person onsite C Person remote") + self.assertEqual(text, f"A Person {areg.affiliation} {areg.country_code} onsite C Person {creg.affiliation} {creg.country_code} remote") + + # Summary stats row: Onsite / Remote / Total (matches registration.ietf.org) + self.assertContains(response, 'Onsite:') + self.assertContains(response, 'Remote:') + self.assertContains(response, 'Total:') + self.assertContains(response, '1') # onsite and remote + self.assertContains(response, '2') # total + + # Chart data embedded in page + chart_json = json.loads(q('#attendees-chart-data').text()) + self.assertEqual(chart_json['type'], [['Onsite', 1], ['Remote', 1]]) def test_proceedings_overview(self): '''Test proceedings IETF Overview page. @@ -9269,27 +9441,25 @@ def test_get_next_sequence(self): self.assertEqual(sequence,1) def test_participants_for_meeting(self): - person_a = PersonFactory() - person_b = PersonFactory() - person_c = PersonFactory() - person_d = PersonFactory() m = MeetingFactory.create(type_id='ietf') - MeetingRegistrationFactory(meeting=m, person=person_a, reg_type='onsite', checkedin=True) - MeetingRegistrationFactory(meeting=m, person=person_b, reg_type='onsite', checkedin=False) - MeetingRegistrationFactory(meeting=m, person=person_c, reg_type='remote') - MeetingRegistrationFactory(meeting=m, person=person_d, reg_type='remote') - AttendedFactory(session__meeting=m, session__type_id='plenary', person=person_c) + areg = RegistrationFactory(meeting=m, checkedin=True, with_ticket={'attendance_type_id': 'onsite'}) + breg = RegistrationFactory(meeting=m, checkedin=False, with_ticket={'attendance_type_id': 'onsite'}) + creg = RegistrationFactory(meeting=m, with_ticket={'attendance_type_id': 'remote'}) + dreg = RegistrationFactory(meeting=m, with_ticket={'attendance_type_id': 'remote'}) + AttendedFactory(session__meeting=m, session__type_id='plenary', person=creg.person) checked_in, attended = participants_for_meeting(m) - self.assertTrue(person_a.pk in checked_in) - self.assertTrue(person_b.pk not in checked_in) - self.assertTrue(person_c.pk in attended) - self.assertTrue(person_d.pk not in attended) + self.assertIn(areg.person.pk, checked_in) + self.assertNotIn(breg.person.pk, checked_in) + self.assertNotIn(areg.person.pk, attended) + self.assertNotIn(breg.person.pk, attended) + self.assertIn(creg.person.pk, attended) + self.assertNotIn(dreg.person.pk, attended) def test_session_attendance(self): meeting = MeetingFactory(type_id='ietf', date=datetime.date(2023, 11, 4), number='118') make_meeting_test_data(meeting=meeting) session = Session.objects.filter(meeting=meeting, group__acronym='mars').first() - regs = MeetingRegistrationFactory.create_batch(3, meeting=meeting) + regs = RegistrationFactory.create_batch(3, meeting=meeting) persons = [reg.person for reg in regs] self.assertEqual(session.attended_set.count(), 0) @@ -9322,7 +9492,7 @@ def test_session_attendance(self): self.assertEqual(r.status_code, 200) self.assertContains(r, '3 attendees') for person in persons: - self.assertContains(r, person.plain_name()) + self.assertContains(r, escape(person.plain_name())) # Test for the "I was there" button. def _test_button(person, expected): @@ -9335,21 +9505,21 @@ def _test_button(person, expected): # person0 is already on the bluesheet _test_button(persons[0], False) # person3 attests he was there - persons.append(MeetingRegistrationFactory(meeting=meeting).person) + persons.append(RegistrationFactory(meeting=meeting).person) # button isn't shown if we're outside the corrections windows meeting.importantdate_set.create(name_id='revsub',date=date_today() - datetime.timedelta(days=20)) _test_button(persons[3], False) # attempt to POST anyway is ignored r = self.client.post(attendance_url) self.assertEqual(r.status_code, 200) - self.assertNotContains(r, persons[3].plain_name()) + self.assertNotContains(r, escape(persons[3].plain_name())) self.assertEqual(session.attended_set.count(), 3) # button is shown, and POST is accepted meeting.importantdate_set.update(name_id='revsub',date=date_today() + datetime.timedelta(days=20)) _test_button(persons[3], True) r = self.client.post(attendance_url) self.assertEqual(r.status_code, 200) - self.assertContains(r, persons[3].plain_name()) + self.assertContains(r, escape(persons[3].plain_name())) self.assertEqual(session.attended_set.count(), 4) # When the meeting is finalized, a bluesheet file is generated, @@ -9393,12 +9563,12 @@ def _test_button(person, expected): def test_bluesheet_data(self): session = SessionFactory(meeting__type_id="ietf") - attended_with_affil = MeetingRegistrationFactory(meeting=session.meeting, affiliation="Somewhere") + attended_with_affil = RegistrationFactory(meeting=session.meeting, affiliation="Somewhere") AttendedFactory(session=session, person=attended_with_affil.person, time="2023-03-13T01:24:00Z") # joined 2nd - attended_no_affil = MeetingRegistrationFactory(meeting=session.meeting) + attended_no_affil = RegistrationFactory(meeting=session.meeting, affiliation="") AttendedFactory(session=session, person=attended_no_affil.person, time="2023-03-13T01:23:00Z") # joined 1st - MeetingRegistrationFactory(meeting=session.meeting) # did not attend - + RegistrationFactory(meeting=session.meeting) # did not attend + data = bluesheet_data(session) self.assertEqual( data, diff --git a/ietf/meeting/urls.py b/ietf/meeting/urls.py index 5da24ddb6f..a038e1cfe6 100644 --- a/ietf/meeting/urls.py +++ b/ietf/meeting/urls.py @@ -1,10 +1,10 @@ -# Copyright The IETF Trust 2007-2024, All Rights Reserved +# Copyright The IETF Trust 2007-2025, All Rights Reserved from django.conf import settings from django.urls import include from django.views.generic import RedirectView -from ietf.meeting import views, views_proceedings +from ietf.meeting import views, views_proceedings, views_session_request from ietf.utils.urls import url class AgendaRedirectView(RedirectView): @@ -15,6 +15,7 @@ def get_redirect_url(self, *args, **kwargs): safe_for_all_meeting_types = [ url(r'^session/(?P[-a-z0-9]+)/?$', views.session_details), + url(r'^session/(?P[-a-z0-9]+)/send_slide_notifications$', views.notify_meetecho_of_all_slides), url(r'^session/(?P\d+)/drafts$', views.add_session_drafts), url(r'^session/(?P\d+)/recordings$', views.add_session_recordings), url(r'^session/(?P\d+)/attendance$', views.session_attendance), @@ -30,7 +31,7 @@ def get_redirect_url(self, *args, **kwargs): url(r'^session/(?P\d+)/doc/%(name)s/remove$' % settings.URL_REGEXPS, views.remove_sessionpresentation), url(r'^session/(?P\d+)\.ics$', views.agenda_ical), url(r'^sessions/(?P[-a-z0-9]+)\.ics$', views.agenda_ical), - url(r'^slidesubmission/(?P\d+)$', views.approve_proposed_slides) + url(r'^slidesubmission/(?P\d+)$', views.approve_proposed_slides), ] @@ -64,7 +65,7 @@ def get_redirect_url(self, *args, **kwargs): type_interim_patterns = [ url(r'^agenda/(?P[A-Za-z0-9-]+)-drafts.pdf$', views.session_draft_pdf), url(r'^agenda/(?P[A-Za-z0-9-]+)-drafts.tgz$', views.session_draft_tarfile), - url(r'^materials/%(document)s(?P\.[a-z0-9]+)$' % settings.URL_REGEXPS, views.materials_document), + url(r'^materials/%(document)s(?P\.[A-Za-z0-9]+)$' % settings.URL_REGEXPS, views.materials_document), url(r'^materials/%(document)s/?$' % settings.URL_REGEXPS, views.materials_document), url(r'^agenda.json$', views.agenda_json) ] @@ -85,7 +86,7 @@ def get_redirect_url(self, *args, **kwargs): url(r'^week-view(?:.html)?/?$', AgendaRedirectView.as_view(pattern_name='agenda', permanent=True)), url(r'^materials(?:.html)?/?$', views.materials), url(r'^request_minutes/?$', views.request_minutes), - url(r'^materials/%(document)s(?P\.[a-z0-9]+)?/?$' % settings.URL_REGEXPS, views.materials_document), + url(r'^materials/%(document)s(?P\.[A-Za-z0-9]+)?/?$' % settings.URL_REGEXPS, views.materials_document), url(r'^session/?$', views.materials_editable_groups), url(r'^proceedings(?:.html)?/?$', views.proceedings), url(r'^proceedings(?:.html)?/finalize/?$', views.finalize_proceedings), @@ -108,6 +109,8 @@ def get_redirect_url(self, *args, **kwargs): url(r'^important-dates.(?Pics)$', views.important_dates), url(r'^proceedings/meetinghosts/edit/', views_proceedings.edit_meetinghosts), url(r'^proceedings/meetinghosts/(?P\d+)/logo/$', views_proceedings.meetinghost_logo), + url(r'^session/request/%(acronym)s/edit/$' % settings.URL_REGEXPS, views_session_request.edit_request), + url(r'^session/request/%(acronym)s/view/$' % settings.URL_REGEXPS, views_session_request.view_request), ] urlpatterns = [ @@ -127,6 +130,13 @@ def get_redirect_url(self, *args, **kwargs): url(r'^upcoming/?$', views.upcoming), url(r'^upcoming\.ics/?$', views.upcoming_ical), url(r'^upcoming\.json/?$', views.upcoming_json), + url(r'^session/request/$', views_session_request.list_view), + url(r'^session/request/%(acronym)s/new/$' % settings.URL_REGEXPS, views_session_request.new_request), + url(r'^session/request/%(acronym)s/approve/$' % settings.URL_REGEXPS, views_session_request.approve_request), + url(r'^session/request/%(acronym)s/no_session/$' % settings.URL_REGEXPS, views_session_request.no_session), + url(r'^session/request/%(acronym)s/cancel/$' % settings.URL_REGEXPS, views_session_request.cancel_request), + url(r'^session/request/%(acronym)s/confirm/$' % settings.URL_REGEXPS, views_session_request.confirm), + url(r'^session/request/status/$', views_session_request.status), url(r'^session/(?P\d+)/agenda_materials$', views.session_materials), url(r'^session/(?P\d+)/cancel/?', views.cancel_session), url(r'^session/(?P\d+)/edit/?', views.edit_session), @@ -140,4 +150,3 @@ def get_redirect_url(self, *args, **kwargs): url(r'^(?P\d+)/', include(safe_for_all_meeting_types)), url(r'^(?Pinterim-[a-z0-9-]+)/', include(safe_for_all_meeting_types)), ] - diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index 9603278399..10ae0d3667 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -2,8 +2,12 @@ # -*- coding: utf-8 -*- import datetime import itertools +from contextlib import suppress +from dataclasses import dataclass + +import jsonschema import os -from hashlib import sha384 +import requests import pytz import subprocess @@ -15,6 +19,7 @@ from django.contrib import messages from django.core.cache import caches from django.core.files.base import ContentFile +from django.db import IntegrityError from django.db.models import OuterRef, Subquery, TextField, Q, Value, Max from django.db.models.functions import Coalesce from django.template.loader import render_to_string @@ -24,16 +29,33 @@ import debug # pyflakes:ignore from ietf.dbtemplate.models import DBTemplate -from ietf.doc.storage_utils import store_bytes, store_str -from ietf.meeting.models import (Session, SchedulingEvent, TimeSlot, - Constraint, SchedTimeSessAssignment, SessionPresentation, Attended) -from ietf.doc.models import Document, State, NewRevisionDocEvent, StateDocEvent +from ietf.doc.storage_utils import store_bytes, store_str, AlreadyExistsError +from ietf.meeting.models import ( + Session, + SchedulingEvent, + TimeSlot, + Constraint, + SchedTimeSessAssignment, + SessionPresentation, + Attended, + Registration, + Meeting, + RegistrationTicket, +) +from ietf.blobdb.models import ResolvedMaterial +from ietf.doc.models import ( + Document, + State, + NewRevisionDocEvent, + StateDocEvent, + StoredObject, +) from ietf.doc.models import DocEvent from ietf.group.models import Group from ietf.group.utils import can_manage_materials from ietf.name.models import SessionStatusName, ConstraintName, DocTypeName from ietf.person.models import Person -from ietf.stats.models import MeetingRegistration +from ietf.utils import markdown from ietf.utils.html import clean_html from ietf.utils.log import log from ietf.utils.timezone import date_today @@ -159,7 +181,7 @@ def bluesheet_data(session): .annotate( affiliation=Coalesce( Subquery( - MeetingRegistration.objects.filter( + Registration.objects.filter( Q(meeting=session.meeting), Q(person=OuterRef("person")) | Q(email=OuterRef("person__email")), ).values("affiliation")[:1] @@ -218,6 +240,7 @@ def save_bluesheet(request, session, file, encoding='utf-8'): save_error = handle_upload_file(file, filename, session.meeting, 'bluesheets', request=request, encoding=encoding) if not save_error: doc.save_with_history([e]) + resolve_uploaded_material(meeting=session.meeting, doc=doc) return save_error @@ -830,6 +853,342 @@ def write_doc_for_session(session, type_id, filename, contents): store_str(type_id, filename.name, contents) return None + +@dataclass +class BlobSpec: + bucket: str + name: str + + +def resolve_one_material( + doc: Document, rev: str | None, ext: str | None +) -> BlobSpec | None: + if doc.type_id is None: + log(f"Cannot resolve a doc with no type: {doc.name}") + return None + + # Get the Document's base name. It may or may not have an extension. + if rev is None: + basename = Path(doc.get_base_name()) + else: + basename = Path(f"{doc.name}-{int(rev):02d}") + + # If the document's file exists, the blob is _always_ named with this stem, + # even if it's different from the original. + blob_stem = Path(f"{doc.name}-{rev or doc.rev}") + + # If we have an extension, either from the URL or the Document's base name, look up + # the blob or file or return 404. N.b. the suffix check needs adjustment to handle + # a bare "." extension when we reach py3.14. + if ext or basename.suffix != "": + if ext: + blob_name = str(blob_stem.with_suffix(ext)) + else: + blob_name = str(blob_stem.with_suffix(basename.suffix)) + + # See if we have a stored object under that name + preferred_blob = ( + StoredObject.objects.exclude_deleted() + .filter(store=doc.type_id, name=blob_name) + .first() + ) + if preferred_blob is not None: + return BlobSpec( + bucket=preferred_blob.store, + name=preferred_blob.name, + ) + # No stored object, fall back to the file system. + filename = Path(doc.get_file_path()) / basename # use basename for file + if filename.is_file(): + return BlobSpec( + bucket=doc.type_id, + name=str(blob_stem.with_suffix(filename.suffix)), + ) + else: + return None + + # No extension has been specified so far, so look one up. + matching_stored_objects = ( + StoredObject.objects.exclude_deleted() + .filter( + store=doc.type_id, + name__startswith=f"{blob_stem}.", # anchor to end with trailing "." + ) + .order_by("name") + ) # orders by suffix + blob_ext_choices = { + Path(stored_obj.name).suffix: stored_obj + for stored_obj in matching_stored_objects + } + + # Short-circuit to return pdf if present + if ".pdf" in blob_ext_choices: + pdf_blob = blob_ext_choices[".pdf"] + return BlobSpec( + bucket=pdf_blob.store, + name=str(blob_stem.with_suffix(".pdf")), + ) + + # Now look for files + filename = Path(doc.get_file_path()) / basename + file_ext_choices = { + # Construct a map from suffix to full filename + fn.suffix: fn.name + for fn in sorted(filename.parent.glob(filename.stem + ".*")) + } + + # Short-circuit to return pdf if we have the file + if ".pdf" in file_ext_choices: + return BlobSpec( + bucket=doc.type_id, + name=str(blob_stem.with_suffix(".pdf")), + ) + + all_exts = set(blob_ext_choices.keys()).union(file_ext_choices.keys()) + if len(all_exts) > 0: + preferred_ext = sorted(all_exts)[0] + if preferred_ext in blob_ext_choices: + preferred_blob = blob_ext_choices[preferred_ext] + return BlobSpec( + bucket=preferred_blob.store, + name=preferred_blob.name, + ) + else: + return BlobSpec( + bucket=doc.type_id, + name=str(blob_stem.with_suffix(preferred_ext)), + ) + + return None + + +def resolve_materials_for_one_meeting(meeting: Meeting): + start_time = timezone.now() + meeting_documents = ( + Document.objects.filter( + type_id__in=settings.MATERIALS_TYPES_SERVED_BY_WORKER + ).filter( + Q(session__meeting=meeting) | Q(proceedingsmaterial__meeting=meeting) + ) + ).distinct() + + resolved = [] + for doc in meeting_documents: + # request by doc name with no rev + blob = resolve_one_material(doc, rev=None, ext=None) + if blob is not None: + resolved.append( + ResolvedMaterial( + name=doc.name, + meeting_number=meeting.number, + bucket=blob.bucket, + blob=blob.name, + ) + ) + # request by doc name + rev + blob = resolve_one_material(doc, rev=doc.rev, ext=None) + if blob is not None: + resolved.append( + ResolvedMaterial( + name=f"{doc.name}-{doc.rev:02}", + meeting_number=meeting.number, + bucket=blob.bucket, + blob=blob.name, + ) + ) + # for other revisions, only need request by doc name + rev + other_revisions = doc.revisions_by_newrevisionevent() + other_revisions.remove(doc.rev) + for rev in other_revisions: + blob = resolve_one_material(doc, rev=rev, ext=None) + if blob is not None: + resolved.append( + ResolvedMaterial( + name=f"{doc.name}-{rev:02}", + meeting_number=meeting.number, + bucket=blob.bucket, + blob=blob.name, + ) + ) + ResolvedMaterial.objects.bulk_create( + resolved, + update_conflicts=True, + unique_fields=["name", "meeting_number"], + update_fields=["bucket", "blob"], + ) + # Warn if any files were updated during the above process + last_update = meeting_documents.aggregate(Max("time"))["time__max"] + if last_update and last_update > start_time: + log( + f"Warning: materials for meeting {meeting.number} " + "changed during ResolvedMaterial update" + ) + +def resolve_uploaded_material(meeting: Meeting, doc: Document): + resolved: list[ResolvedMaterial] = [] + remove = ResolvedMaterial.objects.none() + blob = resolve_one_material(doc, rev=None, ext=None) + if blob is None: + # Versionless file does not exist. Remove the versionless ResolvedMaterial + # if it existed. This is to avoid leaving behind a stale link to a replaced + # version. This comes up e.g. if a ProceedingsMaterial is changed from having + # an uploaded file to being an external URL. + remove = ResolvedMaterial.objects.filter( + name=doc.name, meeting_number=meeting.number + ) + else: + resolved.append( + ResolvedMaterial( + name=doc.name, + meeting_number=meeting.number, + bucket=blob.bucket, + blob=blob.name, + ) + ) + # request by doc name + rev + blob = resolve_one_material(doc, rev=doc.rev, ext=None) + if blob is not None: + resolved.append( + ResolvedMaterial( + name=f"{doc.name}-{doc.rev:02}", + meeting_number=meeting.number, + bucket=blob.bucket, + blob=blob.name, + ) + ) + # Create the new record(s) + ResolvedMaterial.objects.bulk_create( + resolved, + update_conflicts=True, + unique_fields=["name", "meeting_number"], + update_fields=["bucket", "blob"], + ) + # and remove one if necessary (will be a none() queryset if not) + remove.delete() + + +def store_blob_for_one_material_file(doc: Document, rev: str, filepath: Path): + if not settings.ENABLE_BLOBSTORAGE: + raise RuntimeError("Cannot store blobs: ENABLE_BLOBSTORAGE is False") + + bucket = doc.type_id + if bucket not in settings.MATERIALS_TYPES_SERVED_BY_WORKER: + raise ValueError(f"Bucket {bucket} not found for doc {doc.name}.") + blob_stem = f"{doc.name}-{rev}" + suffix = filepath.suffix # includes leading "." + + # Store the file + try: + file_bytes = filepath.read_bytes() + except Exception as err: + log(f"Failed to read {filepath}: {err}") + raise + with suppress(AlreadyExistsError): + store_bytes( + kind=bucket, + name= blob_stem + suffix, + content=file_bytes, + mtime=datetime.datetime.fromtimestamp( + filepath.stat().st_mtime, + tz=datetime.UTC, + ), + allow_overwrite=False, + doc_name=doc.name, + doc_rev=rev, + ) + + # Special case: pre-render markdown into HTML as .md.html + if suffix == ".md": + try: + markdown_source = file_bytes.decode("utf-8") + except UnicodeDecodeError as err: + log(f"Unable to decode {filepath} as UTF-8, treating as latin-1: {err}") + markdown_source = file_bytes.decode("latin-1") + # render the markdown + try: + html = render_to_string( + "minimal.html", + { + "content": markdown.markdown(markdown_source), + "title": blob_stem, + "static_ietf_org": settings.STATIC_IETF_ORG, + }, + ) + except Exception as err: + log(f"Failed to render markdown for {filepath}: {err}") + else: + # Don't overwrite, but don't fail if the blob exists + with suppress(AlreadyExistsError): + store_str( + kind=bucket, + name=blob_stem + ".md.html", + content=html, + allow_overwrite=False, + doc_name=doc.name, + doc_rev=rev, + content_type="text/html;charset=utf-8", + ) + + +def store_blobs_for_one_material_doc(doc: Document): + """Ensure that all files related to a materials Document are in the blob store""" + if doc.type_id not in settings.MATERIALS_TYPES_SERVED_BY_WORKER: + log(f"This method does not handle docs of type {doc.name}") + return + + # Store files for current Document / rev + file_path = Path(doc.get_file_path()) + base_name = Path(doc.get_base_name()) + # .stem would remove directories, so use .with_suffix("") + base_name_stem = str(base_name.with_suffix("")) + if base_name_stem.endswith(".") and base_name.suffix == "": + # In Python 3.14, a trailing "." is a valid suffix, but in prior versions + # it is left as part of the stem. The suffix check ensures that either way, + # only a single "." will be removed. + base_name_stem = base_name_stem[:-1] + # Add any we find without the rev + for file_to_store in file_path.glob(base_name_stem + ".*"): + if not (file_to_store.is_file()): + continue + try: + store_blob_for_one_material_file(doc, doc.rev, file_to_store) + except Exception as err: + log( + f"Failed to store blob for {doc} rev {doc.rev} " + f"from {file_to_store}: {err}" + ) + + # Get other revisions + for rev in doc.revisions_by_newrevisionevent(): + if rev == doc.rev: + continue # already handled this + + # Add some that have the rev + for file_to_store in file_path.glob(doc.name + f"-{rev}.*"): + if not file_to_store.is_file(): + continue + try: + store_blob_for_one_material_file(doc, rev, file_to_store) + except Exception as err: + log( + f"Failed to store blob for {doc} rev {rev} " + f"from {file_to_store}: {err}" + ) + + +def store_blobs_for_one_meeting(meeting: Meeting): + meeting_documents = ( + Document.objects.filter( + type_id__in=settings.MATERIALS_TYPES_SERVED_BY_WORKER + ).filter( + Q(session__meeting=meeting) | Q(proceedingsmaterial__meeting=meeting) + ) + ).distinct() + + for doc in meeting_documents: + store_blobs_for_one_material_doc(doc) + + def create_recording(session, url, title=None, user=None): ''' Creates the Document type=recording, setting external_url and creating @@ -946,13 +1305,14 @@ def get_activity_stats(sdate, edate): data['ffw_update_count'] = ffw_update_count data['ffw_update_percent'] = ffw_update_percent - rfcs = events.filter(type='published_rfc') - data['rfcs'] = rfcs.select_related('doc').select_related('doc__group').select_related('doc__intended_std_level') + rfcs_events = DocEvent.objects.filter(doc__type='rfc', time__gte=sdatetime, time__lt=edatetime) + rfcs = rfcs_events.filter(type='published_rfc') + data['rfcs'] = rfcs.select_related('doc').select_related('doc__group').select_related('doc__std_level') - data['counts'] = {'std': rfcs.filter(doc__intended_std_level__in=('ps', 'ds', 'std')).count(), - 'bcp': rfcs.filter(doc__intended_std_level='bcp').count(), - 'exp': rfcs.filter(doc__intended_std_level='exp').count(), - 'inf': rfcs.filter(doc__intended_std_level='inf').count()} + data['counts'] = {'std': rfcs.filter(doc__std_level__in=('ps', 'ds', 'std')).count(), + 'bcp': rfcs.filter(doc__std_level='bcp').count(), + 'exp': rfcs.filter(doc__std_level='exp').count(), + 'inf': rfcs.filter(doc__std_level='inf').count()} data['new_groups'] = Group.objects.filter( type='wg', @@ -1007,7 +1367,7 @@ def participants_for_meeting(meeting): checked_in = queryset of onsite, checkedin participants values_list('person') attended = queryset of remote participants who attended a session values_list('person') """ - checked_in = meeting.meetingregistration_set.filter(reg_type='onsite', checkedin=True).values_list('person', flat=True).distinct() + checked_in = meeting.registration_set.onsite().filter(checkedin=True).values_list('person', flat=True).distinct() sessions = meeting.session_set.filter(Q(type='plenary') | Q(group__type__in=['wg', 'rg'])) attended = Attended.objects.filter(session__in=sessions).values_list('person', flat=True).distinct() return (checked_in, attended) @@ -1016,15 +1376,50 @@ def participants_for_meeting(meeting): def generate_proceedings_content(meeting, force_refresh=False): """Render proceedings content for a meeting and update cache + Caches its value for 25 hours to ensure that the cache never expires if + we recompute the value daily. + :meeting: meeting whose proceedings should be rendered :force_refresh: true to force regeneration and cache refresh """ - cache = caches["default"] - cache_version = Document.objects.filter(session__meeting__number=meeting.number).aggregate(Max('time'))["time__max"] - # Include proceedings_final in the bare_key so we'll always reflect that accurately, even at the cost of - # a recomputation in the view - bare_key = f"proceedings.{meeting.number}.{cache_version}.final={meeting.proceedings_final}" - cache_key = sha384(bare_key.encode("utf8")).hexdigest() + cache = caches["proceedings"] + key_components = [ + "proceedings", + str(meeting.number), + ] + if meeting.proceedings_final: + # Freeze the cache key once proceedings are finalized. Further changes will + # not be picked up until the cache expires or is refreshed by the + # proceedings_content_refresh_task() + key_components.append("final") + else: + # Build a cache key that changes when materials are modified. For all but drafts, + # use the last modification time of the document. Exclude drafts from this because + # revisions long after the meeting ends will otherwise show up as changes and + # incorrectly invalidate the cache. Instead, include an ordered list of the + # drafts linked to the meeting so adding or removing drafts will trigger a + # recalculation. The list is long but that doesn't matter because we hash it into + # a fixed-length key. + meeting_docs = Document.objects.filter(session__meeting__number=meeting.number) + last_materials_update = ( + meeting_docs.exclude(type_id="draft") + .filter(session__meeting__number=meeting.number) + .aggregate(Max("time"))["time__max"] + ) + draft_names = ( + meeting_docs + .filter(type_id="draft") + .order_by("name") + .values_list("name", flat=True) + ) + key_components += [ + last_materials_update.isoformat() if last_materials_update else "-", + ",".join(draft_names), + ] + + # Key is potentially long, but the "proceedings" cache hashes it to a fixed + # length. If that changes, hash it separately here first. + cache_key = ".".join(key_components) if not force_refresh: cached_content = cache.get(cache_key, None) if cached_content is not None: @@ -1098,7 +1493,7 @@ def area_and_group_acronyms_from_session(s): cache.set( cache_key, rendered_content, - timeout=86400, # one day, in seconds + timeout=3600 + 86400, # one day + one hour, in seconds ) return rendered_content @@ -1177,3 +1572,296 @@ def _format_materials(items): else: not_meeting_groups.append(entry) return meeting_groups, not_meeting_groups + + +import_registration_json_validator = jsonschema.Draft202012Validator( + schema={ + "type": "object", + "properties": { + "objects": { + "type": "object", + "patternProperties": { + # Email address as key (simplified pattern or just allow any key) + ".*": { + "type": "object", + "properties": { + "first_name": {"type": "string"}, + "last_name": {"type": "string"}, + "email": {"type": "string", "format": "email"}, + "affiliation": {"type": "string"}, + "country_code": {"type": "string", "minLength": 2, "maxLength": 2}, + "meeting": {"type": "string"}, + "checkedin": {"type": "boolean"}, + "cancelled": {"type": "boolean"}, + "is_nomcom_volunteer": {"type": "boolean"}, + "tickets": { + "type": "array", + "items": { + "type": "object", + "properties": { + "attendance_type": {"type": "string"}, + "ticket_type": {"type": "string"} + }, + "required": ["attendance_type", "ticket_type"] + } + } + }, + "required": [ + "first_name", "last_name", "email", + "country_code", "meeting", 'affiliation', + "checkedin", "is_nomcom_volunteer", "tickets", + "cancelled", + ] + } + }, + "additionalProperties": False + } + }, + "required": ["objects"] + } +) + + +def get_registration_data(meeting): + '''Retrieve data from registation system for meeting''' + url = settings.REGISTRATION_PARTICIPANTS_API_URL + key = settings.REGISTRATION_PARTICIPANTS_API_KEY + params = {'meeting': meeting.number, 'apikey': key} + try: + response = requests.get(url, params=params, timeout=settings.DEFAULT_REQUESTS_TIMEOUT) + except requests.Timeout as e: + log(f'GET request timed out for [{url}]: {e}') + raise Exception("Timeout retrieving data from registration API") from e + if response.status_code == 200: + try: + decoded = response.json() + except ValueError as e: + raise ValueError(f'Could not decode response from registration API: {e}') + else: + raise Exception(f'Bad response from registration API: {response.status_code}, {response.content[:64]}') + + # validate registration data + import_registration_json_validator.validate(decoded) + return decoded + + +def sync_registration_data(meeting): + """"Sync meeting.Registration with registration system. + + Registration records are created in realtime as people register for a + meeting. This function serves as an audit / reconciliation. Most records are + expected to already exist. The function has been optimized with this in mind. + + - Creates new registrations if they don't exist + - Updates existing registrations if fields differ + - Updates tickets as needed + - Deletes registrations that exist in the database but not in the JSON data + + Returns: + dict: Summary of changes made (created, updated, deleted counts) + """ + reg_data = get_registration_data(meeting) + + # Get the meeting ID from the first registration, the API only deals with one meeting at a time + first_email = next(iter(reg_data['objects'])) + meeting_number = reg_data['objects'][first_email]['meeting'] + try: + Meeting.objects.get(number=meeting_number) + except Meeting.DoesNotExist: + raise Exception(f'meeting does not exist {meeting_number}') + + # Get all existing registrations for this meeting + existing_registrations = meeting.registration_set.all() + existing_emails = set(reg.email for reg in existing_registrations if reg.email) + + # Track changes for reporting + stats = { + 'created': 0, + 'updated': 0, + 'deleted': 0, + 'processed': 0, + } + + # Process registrations from reg_data + reg_emails = set() + for email, data in reg_data['objects'].items(): + stats['processed'] += 1 + reg_emails.add(email) + + # Process this registration + _, action_taken = process_single_registration(data, meeting) + + # Update stats + if action_taken == 'created': + stats['created'] += 1 + elif action_taken == 'updated': + stats['updated'] += 1 + + # Delete registrations that exist in the DB but not in registration data, they've been cancelled + emails_to_delete = existing_emails - reg_emails + if emails_to_delete: + log(f"sync_reg: emails marked for deletion: {emails_to_delete}") + result = Registration.objects.filter( + email__in=emails_to_delete, + meeting=meeting + ).delete() + if 'meeting.Registration' in result[1]: + deleted_count = result[1]['meeting.Registration'] + else: + deleted_count = 0 + stats['deleted'] = deleted_count + # set meeting.attendees + count = Registration.objects.onsite().filter(meeting=meeting, checkedin=True).count() + if meeting.attendees != count: + meeting.attendees = count + meeting.save() + + return stats + + +def process_single_registration(reg_data, meeting): + """ + Process a single registration record - create, update, or leave unchanged as needed. + + Args: + reg_data (dict): Registration data + meeting (obj): The IETF meeting + + Returns: + tuple: (registration, action_taken) + - registration: Registration object + - action_taken: String indicating 'created', 'updated', or None + """ + # import here to avoid circular imports + from ietf.nomcom.models import Volunteer, NomCom + + action_taken = None + fields_updated = False + tickets_modified = False + + # handle deleted + # should not see cancelled records during nightly sync but can see + # them from realtime notifications + if reg_data['cancelled']: + try: + registration = Registration.objects.get(meeting=meeting, email=reg_data['email']) + except Registration.DoesNotExist: + return (None, None) + for ticket in reg_data['tickets']: + target = registration.tickets.filter( + attendance_type__slug=ticket['attendance_type'], + ticket_type__slug=ticket['ticket_type']).first() + if target: + target.delete() + if registration.tickets.count() == 0: + registration.delete() + log(f"sync_reg: cancelled registration {reg_data['email']}") + return (None, 'deleted') + + person = Person.objects.filter(email__address=reg_data['email']).first() + if not person: + log(f"ERROR: meeting registration email unknown {reg_data['email']}") + + registration, created = Registration.objects.get_or_create( + email=reg_data['email'], + meeting=meeting, + defaults={ + 'first_name': reg_data['first_name'], + 'last_name': reg_data['last_name'], + 'person': person, + 'affiliation': reg_data['affiliation'], + 'country_code': reg_data['country_code'], + 'checkedin': reg_data['checkedin'], + } + ) + + # If not created, check if we need to update + if not created: + for field in ['first_name', 'last_name', 'affiliation', 'country_code', 'checkedin']: + if getattr(registration, field) != reg_data[field]: + log(f"sync_reg: found update {reg_data['email']}, {field} different, data from reg: {reg_data}") + setattr(registration, field, reg_data[field]) + fields_updated = True + + if fields_updated: + registration.save() + + # Process tickets - handle counting properly for multiple same-type tickets + # Build count dictionaries for existing and new tickets + existing_ticket_counts = {} + for ticket in registration.tickets.all(): + key = (ticket.attendance_type.slug, ticket.ticket_type.slug) + existing_ticket_counts[key] = existing_ticket_counts.get(key, 0) + 1 + + # Get new tickets from reg_data and count them + reg_data_ticket_counts = {} + for ticket_data in reg_data.get('tickets', []): + key = (ticket_data['attendance_type'], ticket_data['ticket_type']) + reg_data_ticket_counts[key] = reg_data_ticket_counts.get(key, 0) + 1 + + # Calculate tickets to add and remove + all_ticket_types = set(existing_ticket_counts.keys()) | set(reg_data_ticket_counts.keys()) + + for ticket_type in all_ticket_types: + existing_count = existing_ticket_counts.get(ticket_type, 0) + new_count = reg_data_ticket_counts.get(ticket_type, 0) + + # Delete excess tickets + if existing_count > new_count: + tickets_to_delete = existing_count - new_count + # Get all tickets of this type + matching_tickets = registration.tickets.filter( + attendance_type__slug=ticket_type[0], + ticket_type__slug=ticket_type[1] + ).order_by('id') # Use a consistent order for deterministic deletion + + # Delete the required number + log(f"sync_reg: deleting {tickets_to_delete} of {ticket_type[0]}:{ticket_type[1]} of {reg_data['email']}") + for ticket in matching_tickets[:tickets_to_delete]: + ticket.delete() + tickets_modified = True + + # Add missing tickets + elif new_count > existing_count: + tickets_to_add = new_count - existing_count + + # Create the new tickets + log(f"sync_reg: adding {tickets_to_add} of {ticket_type[0]}:{ticket_type[1]} of {reg_data['email']}") + for _ in range(tickets_to_add): + try: + RegistrationTicket.objects.create( + registration=registration, + attendance_type_id=ticket_type[0], + ticket_type_id=ticket_type[1], + ) + tickets_modified = True + except IntegrityError as e: + log(f"Error adding RegistrationTicket {e}") + # handle nomcom volunteer + if reg_data['is_nomcom_volunteer'] and person: + try: + nomcom = NomCom.objects.get(is_accepting_volunteers=True) + except (NomCom.DoesNotExist, NomCom.MultipleObjectsReturned): + nomcom = None + if nomcom: + Volunteer.objects.get_or_create( + nomcom=nomcom, + person=person, + defaults={ + "affiliation": reg_data["affiliation"], + "origin": "registration" + } + ) + + # set action_taken + if created: + log(f"sync_reg: created record. {reg_data['email']}") + action_taken = 'created' + elif fields_updated or tickets_modified: + action_taken = 'updated' + + return registration, action_taken + + +def fetch_attendance_from_meetings(meetings): + return [sync_registration_data(meeting) for meeting in meetings] diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 591b934b58..67a81305b4 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -9,6 +9,7 @@ import json import math import os + import pytz import re import tarfile @@ -27,17 +28,19 @@ from django import forms from django.core.cache import caches +from django.core.files.storage import storages from django.shortcuts import render, redirect, get_object_or_404 from django.http import (HttpResponse, HttpResponseRedirect, HttpResponseForbidden, HttpResponseNotFound, Http404, HttpResponseBadRequest, - JsonResponse, HttpResponseGone, HttpResponseNotAllowed) + JsonResponse, HttpResponseGone, HttpResponseNotAllowed, + FileResponse) from django.conf import settings from django.contrib import messages from django.contrib.auth.decorators import login_required from django.core.exceptions import ValidationError from django.core.files.uploadedfile import SimpleUploadedFile from django.core.validators import URLValidator -from django.urls import reverse,reverse_lazy +from django.urls import reverse, reverse_lazy, NoReverseMatch from django.db.models import F, Max, Q from django.forms.models import modelform_factory, inlineformset_factory from django.template import TemplateDoesNotExist @@ -48,19 +51,27 @@ from django.views.decorators.cache import cache_page from django.views.decorators.csrf import ensure_csrf_cookie, csrf_exempt from django.views.generic import RedirectView +from rest_framework.status import HTTP_404_NOT_FOUND import debug # pyflakes:ignore from ietf.doc.fields import SearchableDocumentsField from ietf.doc.models import Document, State, DocEvent, NewRevisionDocEvent -from ietf.doc.storage_utils import remove_from_storage, retrieve_bytes, store_file +from ietf.doc.storage_utils import ( + remove_from_storage, + retrieve_bytes, + store_file, +) from ietf.group.models import Group from ietf.group.utils import can_manage_session_materials, can_manage_some_groups, can_manage_group from ietf.person.models import Person, User from ietf.ietfauth.utils import role_required, has_role, user_is_person from ietf.mailtrigger.utils import gather_address_lists -from ietf.meeting.models import Meeting, Session, Schedule, FloorPlan, SessionPresentation, TimeSlot, SlideSubmission, Attended +from ietf.meeting.models import Meeting, Session, Schedule, FloorPlan, \ + SessionPresentation, TimeSlot, SlideSubmission, Attended +from ..blobdb.models import ResolvedMaterial from ietf.meeting.models import ImportantDate, SessionStatusName, SchedulingEvent, SchedTimeSessAssignment, Room, TimeSlotTypeName +from ietf.meeting.models import Registration from ietf.meeting.forms import ( CustomDurationField, SwapDaysForm, SwapTimeslotsForm, ImportMinutesForm, TimeSlotCreateForm, TimeSlotEditForm, SessionCancelForm, SessionEditForm ) from ietf.meeting.helpers import get_person_by_email, get_schedule_by_name @@ -82,7 +93,8 @@ finalize, generate_proceedings_content, organize_proceedings_sessions, - sort_accept_tuple, + resolve_uploaded_material, + sort_accept_tuple, store_blobs_for_one_material_doc, ) from ietf.meeting.utils import add_event_info_to_session_qs from ietf.meeting.utils import session_time_for_sorting @@ -97,8 +109,7 @@ from ietf.meeting.utils import get_activity_stats, post_process, create_recording, delete_recording from ietf.meeting.utils import participants_for_meeting, generate_bluesheet, bluesheet_data, save_bluesheet from ietf.message.utils import infer_message -from ietf.name.models import SlideSubmissionStatusName, ProceedingsMaterialTypeName, SessionPurposeName -from ietf.stats.models import MeetingRegistration +from ietf.name.models import SlideSubmissionStatusName, ProceedingsMaterialTypeName, SessionPurposeName, CountryName from ietf.utils import markdown from ietf.utils.decorators import require_api_key from ietf.utils.hedgedoc import Note, NoteError @@ -118,6 +129,11 @@ UploadAgendaForm, UploadBlueSheetForm, UploadMinutesForm, UploadSlidesForm, UploadNarrativeMinutesForm) +from icalendar import Calendar, Event +from ietf.doc.templatetags.ietf_filters import absurl +from ..api.ietf_utils import requires_api_token +from ..blobdb.storage import BlobdbStorage, BlobFile + request_summary_exclude_group_types = ['team'] @@ -137,6 +153,10 @@ def send_interim_change_notice(request, meeting): message.related_groups.add(group) send_mail_message(request, message) +def parse_ical_line_endings(ical): + """Parse icalendar line endings to ensure they are RFC 5545 compliant""" + return re.sub(r'\r(?!\n)|(? 0 - hidden = len(set(filter_params['hide']).intersection(assignment.filter_keywords)) > 0 + if hasattr(assignment, "filter_keywords"): + kw = assignment.filter_keywords + elif isinstance(assignment, dict): + kw = assignment.get("filterKeywords", []) + else: + raise ValueError("Unsupported assignment instance") + shown = len(set(filter_params['show']).intersection(kw)) > 0 + hidden = len(set(filter_params['hide']).intersection(kw)) > 0 return shown and not hidden + +def agenda_ical_ietf(meeting, filt_params, acronym=None, session_id=None): + agenda_data = generate_agenda_data(meeting.number, force_refresh=False) + if acronym: + agenda_data["schedule"] = [ + item + for item in agenda_data["schedule"] + if item["groupAcronym"] == acronym + ] + elif session_id: + agenda_data["schedule"] = [ + item + for item in agenda_data["schedule"] + if item["sessionId"] == session_id + ] + if filt_params is not None: + # Apply the filter + agenda_data["schedule"] = [ + item + for item in agenda_data["schedule"] + if should_include_assignment(filt_params, item) + ] + return render_icalendar_precomp(agenda_data) + + +def agenda_ical_interim(meeting, filt_params, acronym=None, session_id=None): + schedule = get_schedule(meeting) + + if schedule is None and acronym is None and session_id is None: + raise Http404 + + assignments = SchedTimeSessAssignment.objects.filter( + schedule__in=[schedule, schedule.base], + session__on_agenda=True, + ) + assignments = preprocess_assignments_for_agenda(assignments, meeting) + AgendaKeywordTagger(assignments=assignments).apply() + + if filt_params is not None: + # Apply the filter + assignments = [a for a in assignments if should_include_assignment(filt_params, a)] + + if acronym: + assignments = [ a for a in assignments if a.session.group_at_the_time().acronym == acronym ] + elif session_id: + assignments = [ a for a in assignments if a.session_id == int(session_id) ] + + return render_icalendar(schedule, assignments) + + def agenda_ical(request, num=None, acronym=None, session_id=None): """Agenda ical view @@ -2153,42 +2649,20 @@ def agenda_ical(request, num=None, acronym=None, session_id=None): raise Http404 else: meeting = get_meeting(num, type_in=None) # get requested meeting, whatever its type - schedule = get_schedule(meeting) - updated = meeting.updated() - if schedule is None and acronym is None and session_id is None: - raise Http404 - - assignments = SchedTimeSessAssignment.objects.filter( - schedule__in=[schedule, schedule.base], - session__on_agenda=True, - ) - assignments = preprocess_assignments_for_agenda(assignments, meeting) - AgendaKeywordTagger(assignments=assignments).apply() + if isinstance(session_id, str) and session_id.isdigit(): + session_id = int(session_id) try: filt_params = parse_agenda_filter_params(request.GET) except ValueError as e: return HttpResponseBadRequest(str(e)) - if filt_params is not None: - # Apply the filter - assignments = [a for a in assignments if should_include_assignment(filt_params, a)] - - if acronym: - assignments = [ a for a in assignments if a.session.group_at_the_time().acronym == acronym ] - elif session_id: - assignments = [ a for a in assignments if a.session_id == int(session_id) ] - - for a in assignments: - if a.session: - a.session.ical_status = ical_session_status(a) + if meeting.type_id == "ietf": + return agenda_ical_ietf(meeting, filt_params, acronym, session_id) + else: + return agenda_ical_interim(meeting, filt_params, acronym, session_id) - return render(request, "meeting/agenda.ics", { - "schedule": schedule, - "assignments": assignments, - "updated": updated - }, content_type="text/calendar") @cache_page(15 * 60) def agenda_json(request, num=None): @@ -2695,7 +3169,7 @@ def session_attendance(request, session_id, num): raise Http404("Bluesheets not found") cor_cut_off_date = session.meeting.get_submission_correction_date() - today_utc = date_today(datetime.timezone.utc) + today_utc = date_today(datetime.UTC) was_there = False can_add = False if request.user.is_authenticated: @@ -2710,7 +3184,7 @@ def session_attendance(request, session_id, num): was_there = Attended.objects.filter(session=session, person=person).exists() can_add = ( today_utc <= cor_cut_off_date - and MeetingRegistration.objects.filter( + and Registration.objects.filter( meeting=session.meeting, person=person ).exists() and not was_there @@ -2760,7 +3234,10 @@ def upload_session_bluesheets(request, session_id, num): ota = session.official_timeslotassignment() sess_time = ota and ota.timeslot.time if not sess_time: - return HttpResponseGone("Cannot receive uploads for an unscheduled session. Please check the session ID.", content_type="text/plain") + return HttpResponseGone( + "Cannot receive uploads for an unscheduled session. Please check the session ID.", + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + ) save_error = save_bluesheet(request, session, file, encoding=form.file_encoding[file.name]) @@ -2821,12 +3298,13 @@ def upload_session_minutes(request, session_id, num): except SessionNotScheduledError: return HttpResponseGone( "Cannot receive uploads for an unscheduled session. Please check the session ID.", - content_type="text/plain", + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", ) except SaveMaterialsError as err: form.add_error(None, str(err)) else: # no exception -- success! + resolve_uploaded_material(meeting=session.meeting, doc=session.minutes()) messages.success(request, f'Successfully uploaded minutes as revision {session.minutes().rev}.') return redirect('ietf.meeting.views.session_details', num=num, acronym=session.group.acronym) else: @@ -2880,12 +3358,13 @@ def upload_session_narrativeminutes(request, session_id, num): except SessionNotScheduledError: return HttpResponseGone( "Cannot receive uploads for an unscheduled session. Please check the session ID.", - content_type="text/plain", + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", ) except SaveMaterialsError as err: form.add_error(None, str(err)) else: # no exception -- success! + resolve_uploaded_material(meeting=session.meeting, doc=session.narrative_minutes()) messages.success(request, f'Successfully uploaded narrative minutes as revision {session.narrative_minutes().rev}.') return redirect('ietf.meeting.views.session_details', num=num, acronym=session.group.acronym) else: @@ -2978,7 +3457,10 @@ def upload_session_agenda(request, session_id, num): ota = session.official_timeslotassignment() sess_time = ota and ota.timeslot.time if not sess_time: - return HttpResponseGone("Cannot receive uploads for an unscheduled session. Please check the session ID.", content_type="text/plain") + return HttpResponseGone( + "Cannot receive uploads for an unscheduled session. Please check the session ID.", + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + ) if session.meeting.type_id=='ietf': name = 'agenda-%s-%s' % (session.meeting.number, session.group.acronym) @@ -3029,6 +3511,7 @@ def upload_session_agenda(request, session_id, num): form.add_error(None, save_error) else: doc.save_with_history([e]) + resolve_uploaded_material(meeting=session.meeting, doc=doc) messages.success(request, f'Successfully uploaded agenda as revision {doc.rev}.') return redirect('ietf.meeting.views.session_details',num=num,acronym=session.group.acronym) else: @@ -3212,6 +3695,7 @@ def upload_session_slides(request, session_id, num, name=None): else: doc.save_with_history([e]) post_process(doc) + resolve_uploaded_material(meeting=session.meeting, doc=doc) # Send MeetEcho updates even if we had a problem saving - that will keep it in sync with the # SessionPresentation, which was already saved regardless of problems saving the file. @@ -4123,21 +4607,137 @@ def upcoming_ical(request): else: ietfs = [] - meeting_vtz = {meeting.vtimezone() for meeting in meetings} - meeting_vtz.discard(None) - - # icalendar response file should have '\r\n' line endings per RFC5545 - response = render_to_string('meeting/upcoming.ics', { - 'vtimezones': ''.join(sorted(meeting_vtz)), - 'assignments': assignments, - 'ietfs': ietfs, - }, request=request) - response = re.sub("\r(?!\n)|(?= 118: checked_in, attended = participants_for_meeting(meeting) - regs = list(MeetingRegistration.objects.filter(meeting__number=num, reg_type='onsite', checkedin=True)) + regs = list(Registration.objects.onsite().filter(meeting__number=num, checkedin=True)) + onsite_count = len(regs) + regs += [ + reg + for reg in Registration.objects.remote().filter(meeting__number=num).select_related('person') + if reg.person.pk in attended and reg.person.pk not in checked_in + ] + remote_count = len(regs) - onsite_count - for mr in MeetingRegistration.objects.filter(meeting__number=num, reg_type='remote').select_related('person'): - if mr.person.pk in attended and mr.person.pk not in checked_in: - regs.append(mr) + registrations = sorted(regs, key=lambda x: (x.last_name, x.first_name)) - meeting_registrations = sorted(regs, key=lambda x: (x.last_name, x.first_name)) + country_codes = [r.country_code for r in registrations if r.country_code] + stats = { + 'total': onsite_count + remote_count, + 'onsite': onsite_count, + 'remote': remote_count, + } + + code_to_name = dict(CountryName.objects.values_list('slug', 'name')) + country_counts = Counter(code_to_name.get(c, c) for c in country_codes).most_common() + + chart_data = { + 'type': [['Onsite', onsite_count], ['Remote', remote_count]], + 'countries': country_counts, + } else: overview_template = "/meeting/proceedings/%s/attendees.html" % meeting.number try: @@ -4230,8 +4851,10 @@ def proceedings_attendees(request, num=None): return render(request, "meeting/proceedings_attendees.html", { 'meeting': meeting, - 'meeting_registrations': meeting_registrations, + 'registrations': registrations, 'template': template, + 'stats': stats, + 'chart_data': chart_data, }) def proceedings_overview(request, num=None): @@ -4282,11 +4905,17 @@ def api_set_meetecho_recording_name(request): name: the name to use for the recording at meetecho player """ def err(code, text): - return HttpResponse(text, status=code, content_type='text/plain') + return HttpResponse( + text, + status=code, + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + ) if request.method != "POST": return HttpResponseNotAllowed( - content="Method not allowed", content_type="text/plain", permitted_methods=('POST',) + content="Method not allowed", + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + permitted_methods=('POST',), ) session_id = request.POST.get('session_id', None) @@ -4306,7 +4935,11 @@ def err(code, text): session.meetecho_recording_name = name session.save() - return HttpResponse("Done", status=200, content_type='text/plain') + return HttpResponse( + "Done", + status=200, + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + ) @require_api_key @role_required('Recording Manager') @@ -4320,11 +4953,17 @@ def api_set_session_video_url(request): url: The recording url (on YouTube, or whatever) """ def err(code, text): - return HttpResponse(text, status=code, content_type='text/plain') + return HttpResponse( + text, + status=code, + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + ) if request.method != 'POST': return HttpResponseNotAllowed( - content="Method not allowed", content_type="text/plain", permitted_methods=('POST',) + content="Method not allowed", + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + permitted_methods=('POST',), ) # Temporary: fall back to deprecated interface if we have old-style parameters. @@ -4363,7 +5002,11 @@ def err(code, text): time = session.official_timeslotassignment().timeslot.time title = 'Video recording for %s on %s at %s' % (session.group.acronym, time.date(), time.time()) create_recording(session, incoming_url, title=title, user=request.user.person) - return HttpResponse("Done", status=200, content_type='text/plain') + return HttpResponse( + "Done", + status=200, + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + ) def deprecated_api_set_session_video_url(request): @@ -4372,7 +5015,11 @@ def deprecated_api_set_session_video_url(request): Uses meeting/group/item to identify session. """ def err(code, text): - return HttpResponse(text, status=code, content_type='text/plain') + return HttpResponse( + text, + status=code, + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + ) if request.method == 'POST': # parameters: # apikey: the poster's personal API key @@ -4426,7 +5073,11 @@ def err(code, text): else: return err(405, "Method not allowed") - return HttpResponse("Done", status=200, content_type='text/plain') + return HttpResponse( + "Done", + status=200, + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + ) @require_api_key @@ -4478,7 +5129,11 @@ def api_add_session_attendees(request): ) def err(code, text): - return HttpResponse(text, status=code, content_type="text/plain") + return HttpResponse( + text, + status=code, + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + ) if request.method != "POST": return err(405, "Method not allowed") @@ -4531,7 +5186,11 @@ def err(code, text): if save_error: return err(400, save_error) - return HttpResponse("Done", status=200, content_type="text/plain") + return HttpResponse( + "Done", + status=200, + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + ) @require_api_key @@ -4539,7 +5198,11 @@ def err(code, text): @csrf_exempt def api_upload_chatlog(request): def err(code, text): - return HttpResponse(text, status=code, content_type='text/plain') + return HttpResponse( + text, + status=code, + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + ) if request.method != 'POST': return err(405, "Method not allowed") apidata_post = request.POST.get('apidata') @@ -4572,14 +5235,23 @@ def err(code, text): write_doc_for_session(session, 'chatlog', filename, json.dumps(apidata['chatlog'])) e = NewRevisionDocEvent.objects.create(doc=doc, rev=doc.rev, by=request.user.person, type='new_revision', desc='New revision available: %s'%doc.rev) doc.save_with_history([e]) - return HttpResponse("Done", status=200, content_type='text/plain') + resolve_uploaded_material(meeting=session.meeting, doc=doc) + return HttpResponse( + "Done", + status=200, + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + ) @require_api_key @role_required('Recording Manager') @csrf_exempt def api_upload_polls(request): def err(code, text): - return HttpResponse(text, status=code, content_type='text/plain') + return HttpResponse( + text, + status=code, + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + ) if request.method != 'POST': return err(405, "Method not allowed") apidata_post = request.POST.get('apidata') @@ -4612,7 +5284,12 @@ def err(code, text): write_doc_for_session(session, 'polls', filename, json.dumps(apidata['polls'])) e = NewRevisionDocEvent.objects.create(doc=doc, rev=doc.rev, by=request.user.person, type='new_revision', desc='New revision available: %s'%doc.rev) doc.save_with_history([e]) - return HttpResponse("Done", status=200, content_type='text/plain') + resolve_uploaded_material(meeting=session.meeting, doc=doc) + return HttpResponse( + "Done", + status=200, + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + ) @require_api_key @role_required('Recording Manager', 'Secretariat') @@ -4627,11 +5304,17 @@ def api_upload_bluesheet(request): [{'name': 'Name', 'affiliation': 'Organization', }, ...] """ def err(code, text): - return HttpResponse(text, status=code, content_type='text/plain') + return HttpResponse( + text, + status=code, + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + ) if request.method != 'POST': return HttpResponseNotAllowed( - content="Method not allowed", content_type="text/plain", permitted_methods=('POST',) + content="Method not allowed", + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + permitted_methods=('POST',), ) session_id = request.POST.get('session_id', None) @@ -4666,7 +5349,11 @@ def err(code, text): save_err = save_bluesheet(request, session, file) if save_err: return err(400, save_err) - return HttpResponse("Done", status=200, content_type='text/plain') + return HttpResponse( + "Done", + status=200, + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + ) def important_dates(request, num=None, output_format=None): @@ -4692,11 +5379,8 @@ def important_dates(request, num=None, output_format=None): if output_format == 'ics': preprocess_meeting_important_dates(meetings) - ics = render_to_string('meeting/important_dates.ics', { - 'meetings': meetings, - }, request=request) - # icalendar response file should have '\r\n' line endings per RFC5545 - response = HttpResponse(re.sub("\r(?!\n)|(? 0: + messages.success( + request, + f"Notified Meetecho about slides for {','.join(str(s) for s in updated)}", + ) + elif sm.slides_notify_time is not None: + messages.warning( + request, + "No sessions were eligible for Meetecho slides update. Updates are " + f"only sent within {sm.slides_notify_time} before or after the session.", + ) + else: + messages.warning( + request, + "No sessions were eligible for Meetecho slides update. Updates are " + "currently disabled.", + ) + return redirect( + "ietf.meeting.views.session_details", num=meeting.number, acronym=acronym + ) + + def import_session_minutes(request, session_id, num): """Import session minutes from the ietf.notes.org site @@ -5090,11 +5821,12 @@ def import_session_minutes(request, session_id, num): except SessionNotScheduledError: return HttpResponseGone( "Cannot import minutes for an unscheduled session. Please check the session ID.", - content_type="text/plain", + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", ) except SaveMaterialsError as err: form.add_error(None, str(err)) else: + resolve_uploaded_material(meeting=session.meeting, doc=session.minutes()) messages.success(request, f'Successfully imported minutes as revision {session.minutes().rev}.') return redirect('ietf.meeting.views.session_details', num=num, acronym=session.group.acronym) else: @@ -5130,3 +5862,4 @@ def import_session_minutes(request, session_id, num): 'contents_unchanged': not contents_changed, }, ) + diff --git a/ietf/meeting/views_proceedings.py b/ietf/meeting/views_proceedings.py index d1169bff2d..639efa1da4 100644 --- a/ietf/meeting/views_proceedings.py +++ b/ietf/meeting/views_proceedings.py @@ -14,7 +14,7 @@ from ietf.meeting.models import Meeting, MeetingHost from ietf.meeting.helpers import get_meeting from ietf.name.models import ProceedingsMaterialTypeName -from ietf.meeting.utils import handle_upload_file +from ietf.meeting.utils import handle_upload_file, resolve_uploaded_material from ietf.utils.text import xslugify class UploadProceedingsMaterialForm(FileUploadForm): @@ -150,7 +150,7 @@ def save_proceedings_material_doc(meeting, material_type, title, request, file=N if events: doc.save_with_history(events) - + resolve_uploaded_material(meeting, doc) return doc diff --git a/ietf/secr/sreq/views.py b/ietf/meeting/views_session_request.py similarity index 80% rename from ietf/secr/sreq/views.py rename to ietf/meeting/views_session_request.py index eb93168e1c..a1ef74f1b8 100644 --- a/ietf/secr/sreq/views.py +++ b/ietf/meeting/views_session_request.py @@ -1,29 +1,26 @@ -# Copyright The IETF Trust 2013-2022, All Rights Reserved +# Copyright The IETF Trust 2007-2025, All Rights Reserved # -*- coding: utf-8 -*- - import datetime import inflect from collections import defaultdict, OrderedDict from django.conf import settings from django.contrib import messages +from django.core.exceptions import ObjectDoesNotExist from django.db.models import Q from django.shortcuts import render, get_object_or_404, redirect from django.http import Http404 -import debug # pyflakes:ignore - from ietf.group.models import Group, GroupFeatures from ietf.ietfauth.utils import has_role, role_required -from ietf.meeting.models import Meeting, Session, Constraint, ResourceAssociation, SchedulingEvent from ietf.meeting.helpers import get_meeting +from ietf.meeting.models import Session, Meeting, Constraint, ResourceAssociation, SchedulingEvent from ietf.meeting.utils import add_event_info_to_session_qs -from ietf.name.models import SessionStatusName, ConstraintName -from ietf.secr.sreq.forms import (SessionForm, ToolStatusForm, allowed_conflicting_groups, +from ietf.meeting.forms import (SessionRequestStatusForm, SessionRequestForm, allowed_conflicting_groups, JOINT_FOR_SESSION_CHOICES) +from ietf.name.models import SessionStatusName, ConstraintName from ietf.secr.utils.decorators import check_permissions -from ietf.secr.utils.group import get_my_groups from ietf.utils.mail import send_mail from ietf.mailtrigger.utils import gather_address_lists @@ -31,12 +28,25 @@ # Globals # ------------------------------------------------- # TODO: This needs to be replaced with something that pays attention to groupfeatures -AUTHORIZED_ROLES=('WG Chair','WG Secretary','RG Chair','IAB Group Chair','Area Director','Secretariat','Team Chair','IRTF Chair','Program Chair','Program Lead','Program Secretary', 'EDWG Chair') +AUTHORIZED_ROLES = ( + 'WG Chair', + 'WG Secretary', + 'RG Chair', + 'IAB Group Chair', + 'Area Director', + 'Secretariat', + 'Team Chair', + 'IRTF Chair', + 'Program Chair', + 'Program Lead', + 'Program Secretary', + 'EDWG Chair') # ------------------------------------------------- # Helper Functions # ------------------------------------------------- + def check_app_locked(meeting=None): ''' This function returns True if the application is locked to non-secretariat users. @@ -45,6 +55,54 @@ def check_app_locked(meeting=None): meeting = get_meeting(days=14) return bool(meeting.session_request_lock_message) + +def get_lock_message(meeting=None): + ''' + Returns the message to display to non-secretariat users when the tool is locked. + ''' + if not meeting: + meeting = get_meeting(days=14) + return meeting.session_request_lock_message + + +def get_my_groups(user, conclude=False): + ''' + Takes a Django user object (from request) + Returns a list of groups the user has access to. Rules are as follows + secretariat - has access to all groups + area director - has access to all groups in their area + wg chair or secretary - has access to their own group + chair of irtf has access to all irtf groups + + If user=None than all groups are returned. + concluded=True means include concluded groups. Need this to upload materials for groups + after they've been concluded. it happens. + ''' + my_groups = set() + states = ['bof', 'proposed', 'active'] + if conclude: + states.extend(['conclude', 'bof-conc']) + + all_groups = Group.objects.filter(type__features__has_meetings=True, state__in=states).order_by('acronym') + if user is None or has_role(user, 'Secretariat'): + return all_groups + + try: + person = user.person + except ObjectDoesNotExist: + return list() + + for group in all_groups: + if group.role_set.filter(person=person, name__in=('chair', 'secr', 'ad')): + my_groups.add(group) + continue + if group.parent and group.parent.role_set.filter(person=person, name__in=('ad', 'chair')): + my_groups.add(group) + continue + + return list(my_groups) + + def get_initial_session(sessions, prune_conflicts=False): ''' This function takes a queryset of sessions ordered by 'id' for consistency. It returns @@ -97,13 +155,43 @@ def valid_conflict(conflict): initial['joint_for_session_display'] = dict(JOINT_FOR_SESSION_CHOICES)[initial['joint_for_session']] return initial -def get_lock_message(meeting=None): + +def inbound_session_conflicts_as_string(group, meeting): ''' - Returns the message to display to non-secretariat users when the tool is locked. + Takes a Group object and Meeting object and returns a string of other groups which have + a conflict with this one ''' - if not meeting: - meeting = get_meeting(days=14) - return meeting.session_request_lock_message + constraints = group.constraint_target_set.filter(meeting=meeting, name__is_group_conflict=True) + group_set = set(constraints.values_list('source__acronym', flat=True)) # set to de-dupe + group_list = sorted(group_set) # give a consistent order + return ', '.join(group_list) + + +def get_outbound_conflicts(form: SessionRequestForm): + """extract wg conflict constraint data from a SessionForm""" + outbound_conflicts = [] + for conflictname, cfield_id in form.wg_constraint_field_ids(): + conflict_groups = form.cleaned_data[cfield_id] + if len(conflict_groups) > 0: + outbound_conflicts.append(dict(name=conflictname, groups=conflict_groups)) + return outbound_conflicts + + +def save_conflicts(group, meeting, conflicts, name): + ''' + This function takes a Group, Meeting a string which is a list of Groups acronyms (conflicts), + and the constraint name (conflict|conflic2|conflic3) and creates Constraint records + ''' + constraint_name = ConstraintName.objects.get(slug=name) + acronyms = conflicts.replace(',',' ').split() + for acronym in acronyms: + target = Group.objects.get(acronym=acronym) + + constraint = Constraint(source=group, + target=target, + meeting=meeting, + name=constraint_name) + constraint.save() def get_requester_text(person, group): @@ -129,22 +217,6 @@ def get_requester_text(person, group): ) -def save_conflicts(group, meeting, conflicts, name): - ''' - This function takes a Group, Meeting a string which is a list of Groups acronyms (conflicts), - and the constraint name (conflict|conflic2|conflic3) and creates Constraint records - ''' - constraint_name = ConstraintName.objects.get(slug=name) - acronyms = conflicts.replace(',',' ').split() - for acronym in acronyms: - target = Group.objects.get(acronym=acronym) - - constraint = Constraint(source=group, - target=target, - meeting=meeting, - name=constraint_name) - constraint.save() - def send_notification(group, meeting, login, sreq_data, session_data, action): ''' This function generates email notifications for various session request activities. @@ -152,10 +224,10 @@ def send_notification(group, meeting, login, sreq_data, session_data, action): session_data is an array of data from individual session subforms action argument is a string [new|update]. ''' - (to_email, cc_list) = gather_address_lists('session_requested',group=group,person=login) + (to_email, cc_list) = gather_address_lists('session_requested', group=group, person=login) from_email = (settings.SESSION_REQUEST_FROM_EMAIL) subject = '%s - New Meeting Session Request for IETF %s' % (group.acronym, meeting.number) - template = 'sreq/session_request_notification.txt' + template = 'meeting/session_request_notification.txt' # send email context = {} @@ -164,7 +236,7 @@ def send_notification(group, meeting, login, sreq_data, session_data, action): context['meeting'] = meeting context['login'] = login context['header'] = 'A new' - context['requester'] = get_requester_text(login,group) + context['requester'] = get_requester_text(login, group) # update overrides if action == 'update': @@ -174,10 +246,10 @@ def send_notification(group, meeting, login, sreq_data, session_data, action): # if third session requested approval is required # change headers TO=ADs, CC=session-request, submitter and cochairs if len(session_data) > 2: - (to_email, cc_list) = gather_address_lists('session_requested_long',group=group,person=login) + (to_email, cc_list) = gather_address_lists('session_requested_long', group=group, person=login) subject = '%s - Request for meeting session approval for IETF %s' % (group.acronym, meeting.number) - template = 'sreq/session_approval_notification.txt' - #status_text = 'the %s Directors for approval' % group.parent + template = 'meeting/session_approval_notification.txt' + # status_text = 'the %s Directors for approval' % group.parent context['session_lengths'] = [sd['requested_duration'] for sd in session_data] @@ -189,103 +261,188 @@ def send_notification(group, meeting, login, sreq_data, session_data, action): context, cc=cc_list) -def inbound_session_conflicts_as_string(group, meeting): - ''' - Takes a Group object and Meeting object and returns a string of other groups which have - a conflict with this one - ''' - constraints = group.constraint_target_set.filter(meeting=meeting, name__is_group_conflict=True) - group_set = set(constraints.values_list('source__acronym', flat=True)) # set to de-dupe - group_list = sorted(group_set) # give a consistent order - return ', '.join(group_list) + +def session_changed(session): + latest_event = SchedulingEvent.objects.filter(session=session).order_by('-time', '-id').first() + + if latest_event and latest_event.status_id == "schedw" and session.meeting.schedule is not None: + # send an email to iesg-secretariat to alert to change + pass + + +def status_slug_for_new_session(session, session_number): + if session.group.features.acts_like_wg and session_number == 2: + return 'apprw' + return 'schedw' # ------------------------------------------------- # View Functions # ------------------------------------------------- -@check_permissions -def approve(request, acronym): + + +@role_required(*AUTHORIZED_ROLES) +def list_view(request): ''' - This view approves the third session. For use by ADs or Secretariat. + Display list of groups the user has access to. ''' meeting = get_meeting(days=14) - group = get_object_or_404(Group, acronym=acronym) - session = add_event_info_to_session_qs(Session.objects.filter(meeting=meeting, group=group)).filter(current_status='apprw').first() - if session is None: - raise Http404 + # check for locked flag + is_locked = check_app_locked() + if is_locked and not has_role(request.user, 'Secretariat'): + message = get_lock_message() + return render(request, 'meeting/session_request_locked.html', { + 'message': message, + 'meeting': meeting}) - if has_role(request.user,'Secretariat') or group.parent.role_set.filter(name='ad',person=request.user.person): - SchedulingEvent.objects.create( - session=session, - status=SessionStatusName.objects.get(slug='appr'), - by=request.user.person, - ) - session_changed(session) + scheduled_groups = [] + unscheduled_groups = [] - messages.success(request, 'Third session approved') - return redirect('ietf.secr.sreq.views.view', acronym=acronym) - else: - # if an unauthorized user gets here return error - messages.error(request, 'Not authorized to approve the third session') - return redirect('ietf.secr.sreq.views.view', acronym=acronym) + group_types = GroupFeatures.objects.filter(has_meetings=True).values_list('type', flat=True) -@check_permissions -def cancel(request, acronym): - ''' - This view cancels a session request and sends a notification. - To cancel, or withdraw the request set status = deleted. - "canceled" status is used by the secretariat. + my_groups = [g for g in get_my_groups(request.user, conclude=True) if g.type_id in group_types] - NOTE: this function can also be called after a session has been - scheduled during the period when the session request tool is - reopened. In this case be sure to clear the timeslot assignment as well. + sessions_by_group = defaultdict(list) + for s in add_event_info_to_session_qs(Session.objects.filter(meeting=meeting, group__in=my_groups)).filter(current_status__in=['schedw', 'apprw', 'appr', 'sched']): + sessions_by_group[s.group_id].append(s) + + for group in my_groups: + group.meeting_sessions = sessions_by_group.get(group.pk, []) + + if group.pk in sessions_by_group: + # include even if concluded as we need to to see that the + # sessions are there + scheduled_groups.append(group) + else: + if group.state_id not in ['conclude', 'bof-conc']: + # too late for unscheduled if concluded + unscheduled_groups.append(group) + + # warn if there are no associated groups + if not scheduled_groups and not unscheduled_groups: + messages.warning(request, 'The account %s is not associated with any groups. If you have multiple Datatracker accounts you may try another or report a problem to %s' % (request.user, settings.SECRETARIAT_ACTION_EMAIL)) + + # add session status messages for use in template + for group in scheduled_groups: + if not group.features.acts_like_wg or (len(group.meeting_sessions) < 3): + group.status_message = group.meeting_sessions[0].current_status + else: + group.status_message = 'First two sessions: %s, Third session: %s' % (group.meeting_sessions[0].current_status, group.meeting_sessions[2].current_status) + + # add not meeting indicators for use in template + for group in unscheduled_groups: + if any(s.current_status == 'notmeet' for s in group.meeting_sessions): + group.not_meeting = True + + return render(request, 'meeting/session_request_list.html', { + 'is_locked': is_locked, + 'meeting': meeting, + 'scheduled_groups': scheduled_groups, + 'unscheduled_groups': unscheduled_groups}, + ) + + +@role_required('Secretariat') +def status(request): + ''' + This view handles locking and unlocking of the session request tool to the public. ''' meeting = get_meeting(days=14) - group = get_object_or_404(Group, acronym=acronym) - sessions = Session.objects.filter(meeting=meeting,group=group).order_by('id') - login = request.user.person + is_locked = check_app_locked(meeting=meeting) - # delete conflicts - Constraint.objects.filter(meeting=meeting,source=group).delete() + if request.method == 'POST': + button_text = request.POST.get('submit', '') + if button_text == 'Back': + return redirect('ietf.meeting.views_session_request.list_view') - # mark sessions as deleted - for session in sessions: - SchedulingEvent.objects.create( - session=session, - status=SessionStatusName.objects.get(slug='deleted'), - by=request.user.person, - ) - session_changed(session) + form = SessionRequestStatusForm(request.POST) - # clear schedule assignments if already scheduled - session.timeslotassignments.all().delete() + if button_text == 'Lock': + if form.is_valid(): + meeting.session_request_lock_message = form.cleaned_data['message'] + meeting.save() + messages.success(request, 'Session Request Tool is now Locked') + return redirect('ietf.meeting.views_session_request.list_view') - # send notifitcation - (to_email, cc_list) = gather_address_lists('session_request_cancelled',group=group,person=login) - from_email = (settings.SESSION_REQUEST_FROM_EMAIL) - subject = '%s - Cancelling a meeting request for IETF %s' % (group.acronym, meeting.number) - send_mail(request, to_email, from_email, subject, 'sreq/session_cancel_notification.txt', - {'requester':get_requester_text(login,group), - 'meeting':meeting}, cc=cc_list) + elif button_text == 'Unlock': + meeting.session_request_lock_message = '' + meeting.save() + messages.success(request, 'Session Request Tool is now Unlocked') + return redirect('ietf.meeting.views_session_request.list_view') - messages.success(request, 'The %s Session Request has been cancelled' % group.acronym) - return redirect('ietf.secr.sreq.views.main') + else: + if is_locked: + message = get_lock_message() + initial = {'message': message} + form = SessionRequestStatusForm(initial=initial) + else: + form = SessionRequestStatusForm() + return render(request, 'meeting/session_request_status.html', { + 'is_locked': is_locked, + 'form': form}, + ) -def status_slug_for_new_session(session, session_number): - if session.group.features.acts_like_wg and session_number == 2: - return 'apprw' - return 'schedw' +@check_permissions +def new_request(request, acronym): + ''' + This view gathers details for a new session request. The user proceeds to confirm() + to create the request. + ''' + group = get_object_or_404(Group, acronym=acronym) + if len(group.features.session_purposes) == 0: + raise Http404(f'Cannot request sessions for group "{acronym}"') + meeting = get_meeting(days=14) + session_conflicts = dict(inbound=inbound_session_conflicts_as_string(group, meeting)) -def get_outbound_conflicts(form: SessionForm): - """extract wg conflict constraint data from a SessionForm""" - outbound_conflicts = [] - for conflictname, cfield_id in form.wg_constraint_field_ids(): - conflict_groups = form.cleaned_data[cfield_id] - if len(conflict_groups) > 0: - outbound_conflicts.append(dict(name=conflictname, groups=conflict_groups)) - return outbound_conflicts + # check if app is locked + is_locked = check_app_locked() + if is_locked and not has_role(request.user, 'Secretariat'): + messages.warning(request, "The Session Request Tool is closed") + return redirect('ietf.meeting.views_session_request.list_view') + + if request.method == 'POST': + button_text = request.POST.get('submit', '') + if button_text == 'Cancel': + return redirect('ietf.meeting.views_session_request.list_view') + + form = SessionRequestForm(group, meeting, request.POST, notifications_optional=has_role(request.user, "Secretariat")) + if form.is_valid(): + return confirm(request, acronym) + + # the "previous" querystring causes the form to be returned + # pre-populated with data from last meeeting's session request + elif request.method == 'GET' and 'previous' in request.GET: + latest_session = add_event_info_to_session_qs(Session.objects.filter(meeting__type_id='ietf', group=group)).exclude(current_status__in=['notmeet', 'deleted', 'canceled',]).order_by('-meeting__date').first() + if latest_session: + previous_meeting = Meeting.objects.get(number=latest_session.meeting.number) + previous_sessions = add_event_info_to_session_qs(Session.objects.filter(meeting=previous_meeting, group=group)).exclude(current_status__in=['notmeet', 'deleted']).order_by('id') + if not previous_sessions: + messages.warning(request, 'This group did not meet at %s' % previous_meeting) + return redirect('ietf.meeting.views_session_request.new_request', acronym=acronym) + else: + messages.info(request, 'Fetched session info from %s' % previous_meeting) + else: + messages.warning(request, 'Did not find any previous meeting') + return redirect('ietf.meeting.views_session_request.new_request', acronym=acronym) + + initial = get_initial_session(previous_sessions, prune_conflicts=True) + if 'resources' in initial: + initial['resources'] = [x.pk for x in initial['resources']] + form = SessionRequestForm(group, meeting, initial=initial, notifications_optional=has_role(request.user, "Secretariat")) + + else: + initial = {} + form = SessionRequestForm(group, meeting, initial=initial, notifications_optional=has_role(request.user, "Secretariat")) + + return render(request, 'meeting/session_request_form.html', { + 'meeting': meeting, + 'form': form, + 'group': group, + 'is_create': True, + 'session_conflicts': session_conflicts}, + ) @role_required(*AUTHORIZED_ROLES) @@ -295,11 +452,11 @@ def confirm(request, acronym): to confirm for submission. ''' # FIXME: this should be using form.is_valid/form.cleaned_data - invalid input will make it crash - group = get_object_or_404(Group,acronym=acronym) + group = get_object_or_404(Group, acronym=acronym) if len(group.features.session_purposes) == 0: raise Http404(f'Cannot request sessions for group "{acronym}"') meeting = get_meeting(days=14) - form = SessionForm(group, meeting, request.POST, hidden=True, notifications_optional=has_role(request.user, "Secretariat")) + form = SessionRequestForm(group, meeting, request.POST, hidden=True, notifications_optional=has_role(request.user, "Secretariat")) form.is_valid() login = request.user.person @@ -307,8 +464,8 @@ def confirm(request, acronym): # check if request already exists for this group if add_event_info_to_session_qs(Session.objects.filter(group=group, meeting=meeting)).filter(Q(current_status__isnull=True) | ~Q(current_status__in=['deleted', 'notmeet'])): messages.warning(request, 'Sessions for working group %s have already been requested once.' % group.acronym) - return redirect('ietf.secr.sreq.views.main') - + return redirect('ietf.meeting.views_session_request.list_view') + session_data = form.data.copy() # use cleaned_data for the 'bethere' field so we get the Person instances session_data['bethere'] = form.cleaned_data['bethere'] if 'bethere' in form.cleaned_data else [] @@ -318,7 +475,7 @@ def confirm(request, acronym): session_data['joint_for_session_display'] = dict(JOINT_FOR_SESSION_CHOICES)[session_data['joint_for_session']] if form.cleaned_data.get('timeranges'): session_data['timeranges_display'] = [t.desc for t in form.cleaned_data['timeranges']] - session_data['resources'] = [ ResourceAssociation.objects.get(pk=pk) for pk in request.POST.getlist('resources') ] + session_data['resources'] = [ResourceAssociation.objects.get(pk=pk) for pk in request.POST.getlist('resources')] # extract wg conflict constraint data for the view / notifications outbound_conflicts = get_outbound_conflicts(form) @@ -326,7 +483,7 @@ def confirm(request, acronym): button_text = request.POST.get('submit', '') if button_text == 'Cancel': messages.success(request, 'Session Request has been cancelled') - return redirect('ietf.secr.sreq.views.main') + return redirect('ietf.meeting.views_session_request.list_view') if request.method == 'POST' and button_text == 'Submit': # delete any existing session records with status = canceled or notmeet @@ -344,10 +501,10 @@ def confirm(request, acronym): if 'resources' in form.data: new_session.resources.set(session_data['resources']) jfs = form.data.get('joint_for_session', '-1') - if not jfs: # jfs might be '' + if not jfs: # jfs might be '' jfs = '-1' if int(jfs) == count + 1: # count is zero-indexed - groups_split = form.cleaned_data.get('joint_with_groups').replace(',',' ').split() + groups_split = form.cleaned_data.get('joint_with_groups').replace(',', ' ').split() joint = Group.objects.filter(acronym__in=groups_split) new_session.joint_with_groups.set(joint) new_session.save() @@ -388,36 +545,105 @@ def confirm(request, acronym): 'new', ) - status_text = 'IETF Agenda to be scheduled' - messages.success(request, 'Your request has been sent to %s' % status_text) - return redirect('ietf.secr.sreq.views.main') + status_text = 'IETF Agenda to be scheduled' + messages.success(request, 'Your request has been sent to %s' % status_text) + return redirect('ietf.meeting.views_session_request.list_view') + + # POST from request submission + session_conflicts = dict( + outbound=outbound_conflicts, # each is a dict with name and groups as keys + inbound=inbound_session_conflicts_as_string(group, meeting), + ) + if form.cleaned_data.get('third_session'): + messages.warning(request, 'Note: Your request for a third session must be approved by an area director before being submitted to agenda@ietf.org. Click "Submit" below to email an approval request to the area directors') + + return render(request, 'meeting/session_request_confirm.html', { + 'form': form, + 'session': session_data, + 'group': group, + 'meeting': meeting, + 'session_conflicts': session_conflicts}, + ) + + +@role_required(*AUTHORIZED_ROLES) +def view_request(request, acronym, num=None): + ''' + This view displays the session request info + ''' + meeting = get_meeting(num, days=14) + group = get_object_or_404(Group, acronym=acronym) + query = Session.objects.filter(meeting=meeting, group=group) + status_is_null = Q(current_status__isnull=True) + status_allowed = ~Q(current_status__in=("canceled", "notmeet", "deleted")) + sessions = ( + add_event_info_to_session_qs(query) + .filter(status_is_null | status_allowed) + .order_by("id") + ) + + # check if app is locked + is_locked = check_app_locked() + if is_locked: + messages.warning(request, "The Session Request Tool is closed") + + # if there are no session requests yet, redirect to new session request page + if not sessions: + if is_locked: + return redirect('ietf.meeting.views_session_request.list_view') + else: + return redirect('ietf.meeting.views_session_request.new_request', acronym=acronym) + + activities = [{ + 'act_date': e.time.strftime('%b %d, %Y'), + 'act_time': e.time.strftime('%H:%M:%S'), + 'activity': e.status.name, + 'act_by': e.by, + } for e in sessions[0].schedulingevent_set.select_related('status', 'by')] + + # gather outbound conflicts + outbound_dict = OrderedDict() + for obc in group.constraint_source_set.filter(meeting=meeting, name__is_group_conflict=True): + if obc.name.slug not in outbound_dict: + outbound_dict[obc.name.slug] = [] + outbound_dict[obc.name.slug].append(obc.target.acronym) - # POST from request submission session_conflicts = dict( - outbound=outbound_conflicts, # each is a dict with name and groups as keys inbound=inbound_session_conflicts_as_string(group, meeting), + outbound=[dict(name=ConstraintName.objects.get(slug=slug), groups=' '.join(groups)) + for slug, groups in outbound_dict.items()], ) - return render(request, 'sreq/confirm.html', { - 'form': form, - 'session': session_data, + + show_approve_button = False + + # if sessions include a 3rd session waiting approval and the user is a secretariat or AD of the group + # display approve button + if any(s.current_status == 'apprw' for s in sessions): + if has_role(request.user, 'Secretariat') or group.parent.role_set.filter(name='ad', person=request.user.person): + show_approve_button = True + + # build session dictionary (like querydict from new session request form) for use in template + session = get_initial_session(sessions) + + return render(request, 'meeting/session_request_view.html', { + 'can_edit': (not is_locked) or has_role(request.user, 'Secretariat'), + 'can_cancel': (not is_locked) or has_role(request.user, 'Secretariat'), + 'session': session, # legacy processed data + 'sessions': sessions, # actual session instances + 'activities': activities, + 'meeting': meeting, 'group': group, - 'session_conflicts': session_conflicts}, + 'session_conflicts': session_conflicts, + 'show_approve_button': show_approve_button}, ) - -def session_changed(session): - latest_event = SchedulingEvent.objects.filter(session=session).order_by('-time', '-id').first() - - if latest_event and latest_event.status_id == "schedw" and session.meeting.schedule != None: - # send an email to iesg-secretariat to alert to change - pass @check_permissions -def edit(request, acronym, num=None): +def edit_request(request, acronym, num=None): ''' This view allows the user to edit details of the session request ''' - meeting = get_meeting(num,days=14) + meeting = get_meeting(num, days=14) group = get_object_or_404(Group, acronym=acronym) if len(group.features.session_purposes) == 0: raise Http404(f'Cannot request sessions for group "{acronym}"') @@ -443,15 +669,15 @@ def edit(request, acronym, num=None): login = request.user.person first_session = Session() - if(len(sessions) > 0): + if (len(sessions) > 0): first_session = sessions[0] if request.method == 'POST': button_text = request.POST.get('submit', '') if button_text == 'Cancel': - return redirect('ietf.secr.sreq.views.view', acronym=acronym) + return redirect('ietf.meeting.views_session_request.view_request', acronym=acronym) - form = SessionForm(group, meeting, request.POST, initial=initial, notifications_optional=has_role(request.user, "Secretariat")) + form = SessionRequestForm(group, meeting, request.POST, initial=initial, notifications_optional=has_role(request.user, "Secretariat")) if form.is_valid(): if form.has_changed(): changed_session_forms = [sf for sf in form.session_forms.forms_to_keep if sf.has_changed()] @@ -513,11 +739,11 @@ def edit(request, acronym, num=None): if 'resources' in form.changed_data: new_resource_ids = form.cleaned_data['resources'] - new_resources = [ ResourceAssociation.objects.get(pk=a) - for a in new_resource_ids] + new_resources = [ResourceAssociation.objects.get(pk=a) + for a in new_resource_ids] first_session.resources = new_resources - if 'bethere' in form.changed_data and set(form.cleaned_data['bethere'])!=set(initial['bethere']): + if 'bethere' in form.changed_data and set(form.cleaned_data['bethere']) != set(initial['bethere']): first_session.constraints().filter(name='bethere').delete() bethere_cn = ConstraintName.objects.get(slug='bethere') for p in form.cleaned_data['bethere']: @@ -539,7 +765,7 @@ def edit(request, acronym, num=None): # deprecated # log activity - #add_session_activity(group,'Session Request was updated',meeting,user) + # add_session_activity(group,'Session Request was updated',meeting,user) # send notification if form.cleaned_data.get("send_notifications"): @@ -556,7 +782,7 @@ def edit(request, acronym, num=None): ) messages.success(request, 'Session Request updated') - return redirect('ietf.secr.sreq.views.view', acronym=acronym) + return redirect('ietf.meeting.views_session_request.view_request', acronym=acronym) else: # method is not POST # gather outbound conflicts for initial value @@ -567,142 +793,46 @@ def edit(request, acronym, num=None): initial['constraint_{}'.format(slug)] = ' '.join(groups) if not sessions: - return redirect('ietf.secr.sreq.views.new', acronym=acronym) - form = SessionForm(group, meeting, initial=initial, notifications_optional=has_role(request.user, "Secretariat")) + return redirect('ietf.meeting.views_session_request.new_request', acronym=acronym) + form = SessionRequestForm(group, meeting, initial=initial, notifications_optional=has_role(request.user, "Secretariat")) - return render(request, 'sreq/edit.html', { - 'is_locked': is_locked and not has_role(request.user,'Secretariat'), + return render(request, 'meeting/session_request_form.html', { + 'is_locked': is_locked and not has_role(request.user, 'Secretariat'), 'meeting': meeting, 'form': form, 'group': group, + 'is_create': False, 'session_conflicts': session_conflicts}, ) -@role_required(*AUTHORIZED_ROLES) -def main(request): - ''' - Display list of groups the user has access to. - - Template variables - form: a select box populated with unscheduled groups - meeting: the current meeting - scheduled_sessions: - ''' - # check for locked flag - is_locked = check_app_locked() - - if is_locked and not has_role(request.user,'Secretariat'): - message = get_lock_message() - return render(request, 'sreq/locked.html', { - 'message': message}, - ) - - meeting = get_meeting(days=14) - - scheduled_groups = [] - unscheduled_groups = [] - - group_types = GroupFeatures.objects.filter(has_meetings=True).values_list('type', flat=True) - - my_groups = [g for g in get_my_groups(request.user, conclude=True) if g.type_id in group_types] - - sessions_by_group = defaultdict(list) - for s in add_event_info_to_session_qs(Session.objects.filter(meeting=meeting, group__in=my_groups)).filter(current_status__in=['schedw', 'apprw', 'appr', 'sched']): - sessions_by_group[s.group_id].append(s) - - for group in my_groups: - group.meeting_sessions = sessions_by_group.get(group.pk, []) - - if group.pk in sessions_by_group: - # include even if concluded as we need to to see that the - # sessions are there - scheduled_groups.append(group) - else: - if group.state_id not in ['conclude', 'bof-conc']: - # too late for unscheduled if concluded - unscheduled_groups.append(group) - - # warn if there are no associated groups - if not scheduled_groups and not unscheduled_groups: - messages.warning(request, 'The account %s is not associated with any groups. If you have multiple Datatracker accounts you may try another or report a problem to %s' % (request.user, settings.SECRETARIAT_ACTION_EMAIL)) - - # add session status messages for use in template - for group in scheduled_groups: - if not group.features.acts_like_wg or (len(group.meeting_sessions) < 3): - group.status_message = group.meeting_sessions[0].current_status - else: - group.status_message = 'First two sessions: %s, Third session: %s' % (group.meeting_sessions[0].current_status, group.meeting_sessions[2].current_status) - - # add not meeting indicators for use in template - for group in unscheduled_groups: - if any(s.current_status == 'notmeet' for s in group.meeting_sessions): - group.not_meeting = True - - return render(request, 'sreq/main.html', { - 'is_locked': is_locked, - 'meeting': meeting, - 'scheduled_groups': scheduled_groups, - 'unscheduled_groups': unscheduled_groups}, - ) @check_permissions -def new(request, acronym): +def approve_request(request, acronym): ''' - This view gathers details for a new session request. The user proceeds to confirm() - to create the request. + This view approves the third session. For use by ADs or Secretariat. ''' - group = get_object_or_404(Group, acronym=acronym) - if len(group.features.session_purposes) == 0: - raise Http404(f'Cannot request sessions for group "{acronym}"') meeting = get_meeting(days=14) - session_conflicts = dict(inbound=inbound_session_conflicts_as_string(group, meeting)) - - # check if app is locked - is_locked = check_app_locked() - if is_locked and not has_role(request.user,'Secretariat'): - messages.warning(request, "The Session Request Tool is closed") - return redirect('ietf.secr.sreq.views.main') - - if request.method == 'POST': - button_text = request.POST.get('submit', '') - if button_text == 'Cancel': - return redirect('ietf.secr.sreq.views.main') - - form = SessionForm(group, meeting, request.POST, notifications_optional=has_role(request.user, "Secretariat")) - if form.is_valid(): - return confirm(request, acronym) + group = get_object_or_404(Group, acronym=acronym) - # the "previous" querystring causes the form to be returned - # pre-populated with data from last meeeting's session request - elif request.method == 'GET' and 'previous' in request.GET: - latest_session = add_event_info_to_session_qs(Session.objects.filter(meeting__type_id='ietf', group=group)).exclude(current_status__in=['notmeet', 'deleted', 'canceled',]).order_by('-meeting__date').first() - if latest_session: - previous_meeting = Meeting.objects.get(number=latest_session.meeting.number) - previous_sessions = add_event_info_to_session_qs(Session.objects.filter(meeting=previous_meeting, group=group)).exclude(current_status__in=['notmeet', 'deleted']).order_by('id') - if not previous_sessions: - messages.warning(request, 'This group did not meet at %s' % previous_meeting) - return redirect('ietf.secr.sreq.views.new', acronym=acronym) - else: - messages.info(request, 'Fetched session info from %s' % previous_meeting) - else: - messages.warning(request, 'Did not find any previous meeting') - return redirect('ietf.secr.sreq.views.new', acronym=acronym) + session = add_event_info_to_session_qs(Session.objects.filter(meeting=meeting, group=group)).filter(current_status='apprw').first() + if session is None: + raise Http404 - initial = get_initial_session(previous_sessions, prune_conflicts=True) - if 'resources' in initial: - initial['resources'] = [x.pk for x in initial['resources']] - form = SessionForm(group, meeting, initial=initial, notifications_optional=has_role(request.user, "Secretariat")) + if has_role(request.user, 'Secretariat') or group.parent.role_set.filter(name='ad', person=request.user.person): + SchedulingEvent.objects.create( + session=session, + status=SessionStatusName.objects.get(slug='appr'), + by=request.user.person, + ) + session_changed(session) + messages.success(request, 'Third session approved') + return redirect('ietf.meeting.views_session_request.view_request', acronym=acronym) else: - initial={} - form = SessionForm(group, meeting, initial=initial, notifications_optional=has_role(request.user, "Secretariat")) + # if an unauthorized user gets here return error + messages.error(request, 'Not authorized to approve the third session') + return redirect('ietf.meeting.views_session_request.view_request', acronym=acronym) - return render(request, 'sreq/new.html', { - 'meeting': meeting, - 'form': form, - 'group': group, - 'session_conflicts': session_conflicts}, - ) @check_permissions def no_session(request, acronym): @@ -722,7 +852,7 @@ def no_session(request, acronym): # skip if state is already notmeet if add_event_info_to_session_qs(Session.objects.filter(group=group, meeting=meeting)).filter(current_status='notmeet'): messages.info(request, 'The group %s is already marked as not meeting' % group.acronym) - return redirect('ietf.secr.sreq.views.main') + return redirect('ietf.meeting.views_session_request.list_view') session = Session.objects.create( group=group, @@ -740,125 +870,62 @@ def no_session(request, acronym): session_changed(session) # send notification - (to_email, cc_list) = gather_address_lists('session_request_not_meeting',group=group,person=login) + (to_email, cc_list) = gather_address_lists('session_request_not_meeting', group=group, person=login) from_email = (settings.SESSION_REQUEST_FROM_EMAIL) subject = '%s - Not having a session at IETF %s' % (group.acronym, meeting.number) - send_mail(request, to_email, from_email, subject, 'sreq/not_meeting_notification.txt', - {'login':login, - 'group':group, - 'meeting':meeting}, cc=cc_list) + send_mail(request, to_email, from_email, subject, 'meeting/session_not_meeting_notification.txt', + {'login': login, + 'group': group, + 'meeting': meeting}, cc=cc_list) # deprecated? # log activity - #text = 'A message was sent to notify not having a session at IETF %d' % meeting.meeting_num - #add_session_activity(group,text,meeting,request.person) + # text = 'A message was sent to notify not having a session at IETF %d' % meeting.meeting_num + # add_session_activity(group,text,meeting,request.person) # redirect messages.success(request, 'A message was sent to notify not having a session at IETF %s' % meeting.number) - return redirect('ietf.secr.sreq.views.main') - -@role_required('Secretariat') -def tool_status(request): - ''' - This view handles locking and unlocking of the tool to the public. - ''' - meeting = get_meeting(days=14) - is_locked = check_app_locked(meeting=meeting) - - if request.method == 'POST': - button_text = request.POST.get('submit', '') - if button_text == 'Back': - return redirect('ietf.secr.sreq.views.main') - - form = ToolStatusForm(request.POST) - - if button_text == 'Lock': - if form.is_valid(): - meeting.session_request_lock_message = form.cleaned_data['message'] - meeting.save() - messages.success(request, 'Session Request Tool is now Locked') - return redirect('ietf.secr.sreq.views.main') - - elif button_text == 'Unlock': - meeting.session_request_lock_message = '' - meeting.save() - messages.success(request, 'Session Request Tool is now Unlocked') - return redirect('ietf.secr.sreq.views.main') - - else: - if is_locked: - message = get_lock_message() - initial = {'message': message} - form = ToolStatusForm(initial=initial) - else: - form = ToolStatusForm() + return redirect('ietf.meeting.views_session_request.list_view') - return render(request, 'sreq/tool_status.html', { - 'is_locked': is_locked, - 'form': form}, - ) -@role_required(*AUTHORIZED_ROLES) -def view(request, acronym, num = None): +@check_permissions +def cancel_request(request, acronym): ''' - This view displays the session request info + This view cancels a session request and sends a notification. + To cancel, or withdraw the request set status = deleted. + "canceled" status is used by the secretariat. + + NOTE: this function can also be called after a session has been + scheduled during the period when the session request tool is + reopened. In this case be sure to clear the timeslot assignment as well. ''' - meeting = get_meeting(num,days=14) + meeting = get_meeting(days=14) group = get_object_or_404(Group, acronym=acronym) - sessions = add_event_info_to_session_qs(Session.objects.filter(meeting=meeting, group=group)).filter(Q(current_status__isnull=True) | ~Q(current_status__in=('canceled','notmeet','deleted'))).order_by('id') - - # check if app is locked - is_locked = check_app_locked() - if is_locked: - messages.warning(request, "The Session Request Tool is closed") - - # if there are no session requests yet, redirect to new session request page - if not sessions: - if is_locked: - return redirect('ietf.secr.sreq.views.main') - else: - return redirect('ietf.secr.sreq.views.new', acronym=acronym) - - activities = [{ - 'act_date': e.time.strftime('%b %d, %Y'), - 'act_time': e.time.strftime('%H:%M:%S'), - 'activity': e.status.name, - 'act_by': e.by, - } for e in sessions[0].schedulingevent_set.select_related('status', 'by')] - - # gather outbound conflicts - outbound_dict = OrderedDict() - for obc in group.constraint_source_set.filter(meeting=meeting, name__is_group_conflict=True): - if obc.name.slug not in outbound_dict: - outbound_dict[obc.name.slug] = [] - outbound_dict[obc.name.slug].append(obc.target.acronym) - - session_conflicts = dict( - inbound=inbound_session_conflicts_as_string(group, meeting), - outbound=[dict(name=ConstraintName.objects.get(slug=slug), groups=' '.join(groups)) - for slug, groups in outbound_dict.items()], - ) + sessions = Session.objects.filter(meeting=meeting, group=group).order_by('id') + login = request.user.person - show_approve_button = False + # delete conflicts + Constraint.objects.filter(meeting=meeting, source=group).delete() - # if sessions include a 3rd session waiting approval and the user is a secretariat or AD of the group - # display approve button - if any(s.current_status == 'apprw' for s in sessions): - if has_role(request.user,'Secretariat') or group.parent.role_set.filter(name='ad',person=request.user.person): - show_approve_button = True + # mark sessions as deleted + for session in sessions: + SchedulingEvent.objects.create( + session=session, + status=SessionStatusName.objects.get(slug='deleted'), + by=request.user.person, + ) + session_changed(session) - # build session dictionary (like querydict from new session request form) for use in template - session = get_initial_session(sessions) + # clear schedule assignments if already scheduled + session.timeslotassignments.all().delete() - return render(request, 'sreq/view.html', { - 'can_edit': (not is_locked) or has_role(request.user, 'Secretariat'), - 'can_cancel': (not is_locked) or has_role(request.user, 'Secretariat'), - 'session': session, # legacy processed data - 'sessions': sessions, # actual session instances - 'activities': activities, - 'meeting': meeting, - 'group': group, - 'session_conflicts': session_conflicts, - 'show_approve_button': show_approve_button}, - ) + # send notifitcation + (to_email, cc_list) = gather_address_lists('session_request_cancelled', group=group, person=login) + from_email = (settings.SESSION_REQUEST_FROM_EMAIL) + subject = '%s - Cancelling a meeting request for IETF %s' % (group.acronym, meeting.number) + send_mail(request, to_email, from_email, subject, 'meeting/session_cancel_notification.txt', + {'requester': get_requester_text(login, group), + 'meeting': meeting}, cc=cc_list) + messages.success(request, 'The %s Session Request has been cancelled' % group.acronym) + return redirect('ietf.meeting.views_session_request.list_view') diff --git a/ietf/message/admin.py b/ietf/message/admin.py index 250e1eb596..6a876cdc70 100644 --- a/ietf/message/admin.py +++ b/ietf/message/admin.py @@ -27,7 +27,8 @@ def queryset(self, request, queryset): class MessageAdmin(admin.ModelAdmin): - list_display = ["sent_status", "subject", "by", "time", "groups"] + list_display = ["sent_status", "display_subject", "by", "time", "groups"] + list_display_links = ["display_subject"] search_fields = ["subject", "body"] raw_id_fields = ["by", "related_groups", "related_docs"] list_filter = [ @@ -37,6 +38,10 @@ class MessageAdmin(admin.ModelAdmin): ordering = ["-time"] actions = ["retry_send"] + @admin.display(description="Subject", empty_value="(no subject)") + def display_subject(self, instance): + return instance.subject or None # None triggers the empty_value + def groups(self, instance): return ", ".join(g.acronym for g in instance.related_groups.all()) diff --git a/ietf/message/tests.py b/ietf/message/tests.py index a677d5477e..e1bad9a1e6 100644 --- a/ietf/message/tests.py +++ b/ietf/message/tests.py @@ -1,7 +1,7 @@ # Copyright The IETF Trust 2013-2020, All Rights Reserved # -*- coding: utf-8 -*- import datetime -import mock +from unittest import mock from smtplib import SMTPException diff --git a/ietf/message/views.py b/ietf/message/views.py index e4cca63017..355dcdd8d2 100644 --- a/ietf/message/views.py +++ b/ietf/message/views.py @@ -1,3 +1,4 @@ +# Copyright The IETF Trust 2013-2025, All Rights Reserved from django.shortcuts import render, get_object_or_404 from ietf.message.models import Message diff --git a/ietf/middleware.py b/ietf/middleware.py index a4b7a0d24c..fa2e8efd0c 100644 --- a/ietf/middleware.py +++ b/ietf/middleware.py @@ -8,6 +8,7 @@ from django.http import HttpResponsePermanentRedirect from ietf.utils.log import log, exc_parts from ietf.utils.mail import log_smtp_exception +from opentelemetry.propagate import inject import re import smtplib import unicodedata @@ -99,3 +100,12 @@ def add_header(request): return response return add_header + +def add_otel_traceparent_header(get_response): + """Middleware to add the OpenTelemetry traceparent id header to the response""" + def add_header(request): + response = get_response(request) + inject(response) + return response + + return add_header diff --git a/ietf/name/admin.py b/ietf/name/admin.py index 2458da37d9..b89d6d141c 100644 --- a/ietf/name/admin.py +++ b/ietf/name/admin.py @@ -3,6 +3,7 @@ from ietf.name.models import ( AgendaTypeName, + AttendanceTypeName, BallotPositionName, ConstraintName, ContinentName, @@ -29,6 +30,7 @@ LiaisonStatementTagName, MeetingTypeName, NomineePositionStateName, + RegistrationTicketTypeName, ReviewRequestStateName, ReviewResultName, ReviewTypeName, @@ -55,6 +57,7 @@ from ietf.stats.models import CountryAlias +from ietf.utils.admin import SaferTabularInline class NameAdmin(admin.ModelAdmin): @@ -84,7 +87,7 @@ class GroupTypeNameAdmin(NameAdmin): admin.site.register(GroupTypeName, GroupTypeNameAdmin) -class CountryAliasInline(admin.TabularInline): +class CountryAliasInline(SaferTabularInline): model = CountryAlias extra = 1 @@ -137,6 +140,7 @@ class ProceedingsMaterialTypeNameAdmin(NameAdmin): admin.site.register(AgendaFilterTypeName, NameAdmin) admin.site.register(AgendaTypeName, NameAdmin) admin.site.register(AppealArtifactTypeName, NameAdmin) +admin.site.register(AttendanceTypeName, NameAdmin) admin.site.register(BallotPositionName, NameAdmin) admin.site.register(ConstraintName, NameAdmin) admin.site.register(ContinentName, NameAdmin) @@ -158,6 +162,7 @@ class ProceedingsMaterialTypeNameAdmin(NameAdmin): admin.site.register(LiaisonStatementTagName, NameAdmin) admin.site.register(MeetingTypeName, NameAdmin) admin.site.register(NomineePositionStateName, NameAdmin) +admin.site.register(RegistrationTicketTypeName, NameAdmin) admin.site.register(ReviewRequestStateName, NameAdmin) admin.site.register(ReviewAssignmentStateName, NameAdmin) admin.site.register(ReviewResultName, NameAdmin) diff --git a/ietf/name/fixtures/names.json b/ietf/name/fixtures/names.json index 96273fbc35..798fa9178e 100644 --- a/ietf/name/fixtures/names.json +++ b/ietf/name/fixtures/names.json @@ -650,7 +650,7 @@ }, { "fields": { - "desc": "4.2.1. Call for Adoption by WG Issued\r\n\r\n The \"Call for Adoption by WG Issued\" state should be used to indicate when an I-D is being considered for adoption by an IETF WG. An I-D that is in this state is actively being considered for adoption and has not yet achieved consensus, preference, or selection in the WG.\r\n\r\n This state may be used to describe an I-D that someone has asked a WG to consider for adoption, if the WG Chair has agreed with the request. This state may also be used to identify an I-D that a WG Chair asked an author to write specifically for consideration as a candidate WG item [WGDTSPEC], and/or an I-D that is listed as a 'candidate draft' in the WG's charter.\r\n\r\n Under normal conditions, it should not be possible for an I-D to be in the \"Call for Adoption by WG Issued\" state in more than one working group at the same time. This said, it is not uncommon for authors to \"shop\" their I-Ds to more than one WG at a time, with the hope of getting their documents adopted somewhere.\r\n\r\n After this state is implemented in the Datatracker, an I-D that is in the \"Call for Adoption by WG Issued\" state will not be able to be \"shopped\" to any other WG without the consent of the WG Chairs and the responsible ADs impacted by the shopping.\r\n\r\n Note that Figure 1 includes an arc leading from this state to outside of the WG state machine. This illustrates that some I-Ds that are considered do not get adopted as WG drafts. An I-D that is not adopted as a WG draft will transition out of the WG state machine and revert back to having no stream-specific state; however, the status change history log of the I-D will record that the I-D was previously in the \"Call for Adoption by WG Issued\" state.", + "desc": "A call for adoption of the individual submission document has been issued by the Working Group (WG) chairs. This call is still running but the WG has not yet reached consensus for adoption.", "name": "Call For Adoption By WG Issued", "next_states": [ 36, @@ -666,7 +666,7 @@ }, { "fields": { - "desc": "4.2.2. Adopted by a WG\r\n\r\n The \"Adopted by a WG\" state describes an individual submission I-D that an IETF WG has agreed to adopt as one of its WG drafts.\r\n\r\n WG Chairs who use this state will be able to clearly indicate when their WGs adopt individual submission I-Ds. This will facilitate the Datatracker's ability to correctly capture \"Replaces\" information for WG drafts and correct \"Replaced by\" information for individual submission I-Ds that have been replaced by WG drafts.\r\n\r\n This state is needed because the Datatracker uses the filename of an I-D as a key to search its database for status information about the I-D, and because the filename of a WG I-D is supposed to be different from the filename of an individual submission I-D. The filename of an individual submission I-D will typically be formatted as 'draft-author-wgname-topic-nn'.\r\n\r\n The filename of a WG document is supposed to be formatted as 'draft- ietf-wgname-topic-nn'.\r\n\r\n An individual I-D that is adopted by a WG may take weeks or months to be resubmitted by the author as a new (version-00) WG draft. If the \"Adopted by a WG\" state is not used, the Datatracker has no way to determine that an I-D has been adopted until a new version of the I-D is submitted to the WG by the author and until the I-D is approved for posting by a WG Chair.", + "desc": "The individual submission document has been adopted by the Working Group (WG), but some administrative matter still needs to be completed (e.g., a WG document replacing this document with the typical naming convention of 'draft-ietf-wgname-topic-nn' has not yet been submitted).", "name": "Adopted by a WG", "next_states": [ 38 @@ -681,7 +681,7 @@ }, { "fields": { - "desc": "4.2.3. Adopted for WG Info Only\r\n\r\n The \"Adopted for WG Info Only\" state describes a document that contains useful information for the WG that adopted it, but the document is not intended to be published as an RFC. The WG will not actively develop the contents of the I-D or progress it for publication as an RFC. The only purpose of the I-D is to provide information for internal use by the WG.", + "desc": "The document is adopted by the Working Group (WG) for its internal use. The WG has decided that it will not pursue publication of it as an RFC.", "name": "Adopted for WG Info Only", "next_states": [], "order": 3, @@ -694,7 +694,7 @@ }, { "fields": { - "desc": "4.2.4. WG Document\r\n\r\n The \"WG Document\" state describes an I-D that has been adopted by an IETF WG and is being actively developed.\r\n\r\n A WG Chair may transition an I-D into the \"WG Document\" state at any time as long as the I-D is not being considered or developed in any other WG.\r\n\r\n Alternatively, WG Chairs may rely upon new functionality to be added to the Datatracker to automatically move version-00 drafts into the \"WG Document\" state as described in Section 4.1.\r\n\r\n Under normal conditions, it should not be possible for an I-D to be in the \"WG Document\" state in more than one WG at a time. This said, I-Ds may be transferred from one WG to another with the consent of the WG Chairs and the responsible ADs.", + "desc": "The document has been identified as a Working Group (WG) document and is under development per Section 7.2 of RFC2418.", "name": "WG Document", "next_states": [ 39, @@ -712,7 +712,7 @@ }, { "fields": { - "desc": "4.2.5. Parked WG Document\r\n\r\n A \"Parked WG Document\" is an I-D that has lost its author or editor, is waiting for another document to be written or for a review to be completed, or cannot be progressed by the working group for some other reason.\r\n\r\n Some of the annotation tags described in Section 4.3 may be used in conjunction with this state to indicate why an I-D has been parked, and/or what may need to happen for the I-D to be un-parked.\r\n\r\n Parking a WG draft will not prevent it from expiring; however, this state can be used to indicate why the I-D has stopped progressing in the WG.\r\n\r\n A \"Parked WG Document\" that is not expired may be transferred from one WG to another with the consent of the WG Chairs and the responsible ADs.", + "desc": "The Working Group (WG) document is in a temporary state where it will not be actively developed. The reason for the pause is explained via a datatracker comments section.", "name": "Parked WG Document", "next_states": [ 38 @@ -727,7 +727,7 @@ }, { "fields": { - "desc": "4.2.6. Dead WG Document\r\n\r\n A \"Dead WG Document\" is an I-D that has been abandoned. Note that 'Dead' is not always a final state for a WG I-D. If consensus is subsequently achieved, a \"Dead WG Document\" may be resurrected. A \"Dead WG Document\" that is not resurrected will eventually expire.\r\n\r\n Note that an I-D that is declared to be \"Dead\" in one WG and that is not expired may be transferred to a non-dead state in another WG with the consent of the WG Chairs and the responsible ADs.", + "desc": "The Working Group (WG) document has been abandoned by the WG. No further development is planned in this WG. A decision to resume work on this document and move it out of this state is possible.", "name": "Dead WG Document", "next_states": [ 38 @@ -742,7 +742,7 @@ }, { "fields": { - "desc": "4.2.7. In WG Last Call\r\n\r\n A document \"In WG Last Call\" is an I-D for which a WG Last Call (WGLC) has been issued and is in progress.\r\n\r\n Note that conducting a WGLC is an optional part of the IETF WG process, per Section 7.4 of RFC 2418 [RFC2418].\r\n\r\n If a WG Chair decides to conduct a WGLC on an I-D, the \"In WG Last Call\" state can be used to track the progress of the WGLC. The Chair may configure the Datatracker to send a WGLC message to one or more mailing lists when the Chair moves the I-D into this state. The WG Chair may also be able to select a different set of mailing lists for a different document undergoing a WGLC; some documents may deserve coordination with other WGs.\r\n\r\n A WG I-D in this state should remain \"In WG Last Call\" until the WG Chair moves it to another state. The WG Chair may configure the Datatracker to send an e-mail after a specified period of time to remind or 'nudge' the Chair to conclude the WGLC and to determine the next state for the document.\r\n\r\n It is possible for one WGLC to lead into another WGLC for the same document. For example, an I-D that completed a WGLC as an \"Informational\" document may need another WGLC if a decision is taken to convert the I-D into a Standards Track document.", + "desc": "The Working Group (WG) document is currently subject to an active WG Last Call (WGLC) review per Section 7.4 of RFC2418.", "name": "In WG Last Call", "next_states": [ 38, @@ -759,7 +759,7 @@ }, { "fields": { - "desc": "4.2.8. Waiting for WG Chair Go-Ahead\r\n\r\n A WG Chair may wish to place an I-D that receives a lot of comments during a WGLC into the \"Waiting for WG Chair Go-Ahead\" state. This state describes an I-D that has undergone a WGLC; however, the Chair is not yet ready to call consensus on the document.\r\n\r\n If comments from the WGLC need to be responded to, or a revision to the I-D is needed, the Chair may place an I-D into this state until all of the WGLC comments are adequately addressed and the (possibly revised) document is in the I-D repository.", + "desc": "The Working Group (WG) document has completed Working Group Last Call (WGLC), but the WG chairs are not yet ready to call consensus on the document. The reasons for this may include comments from the WGLC need to be responded to, or a revision to the document is needed.", "name": "Waiting for WG Chair Go-Ahead", "next_states": [ 41, @@ -775,7 +775,7 @@ }, { "fields": { - "desc": "4.2.9. WG Consensus: Waiting for Writeup\r\n\r\n A document in the \"WG Consensus: Waiting for Writeup\" state has essentially completed its development within the working group, and is nearly ready to be sent to the IESG for publication. The last thing to be done is the preparation of a protocol writeup by a Document Shepherd. The IESG requires that a document shepherd writeup be completed before publication of the I-D is requested. The IETF document shepherding process and the role of a WG Document Shepherd is described in RFC 4858 [RFC4858]\r\n\r\n A WG Chair may call consensus on an I-D without a formal WGLC and transition an I-D that was in the \"WG Document\" state directly into this state.\r\n\r\n The name of this state includes the words \"Waiting for Writeup\" because a good document shepherd writeup takes time to prepare.", + "desc": "The Working Group (WG) document has consensus to proceed to publication. However, the document is waiting for a document shepherd write-up per RFC4858.", "name": "WG Consensus: Waiting for Write-Up", "next_states": [ 44 @@ -790,7 +790,7 @@ }, { "fields": { - "desc": "4.2.10. Submitted to IESG for Publication\r\n\r\n This state describes a WG document that has been submitted to the IESG for publication and that has not been sent back to the working group for revision.\r\n\r\n An I-D in this state may be under review by the IESG, it may have been approved and be in the RFC Editor's queue, or it may have been published as an RFC. Other possibilities exist too. The document may be \"Dead\" (in the IESG state machine) or in a \"Do Not Publish\" state.", + "desc": "The Working Group (WG) document has been submitted to the Internet Engineering Steering Group (IESG) for evaluation and publication per Section 7.4 of RFC2418. See the “IESG State” or “RFC Editor State” for further details on the state of the document.", "name": "Submitted to IESG for Publication", "next_states": [ 38 @@ -2020,7 +2020,7 @@ }, { "fields": { - "desc": "The document has been marked as a candidate for WG adoption by the WG Chair. This state can be used before a call for adoption is issued (and the document is put in the \"Call For Adoption By WG Issued\" state), to indicate that the document is in the queue for a call for adoption, even if none has been issued yet.", + "desc": "The individual submission document has been marked by the Working Group (WG) chairs as a candidate for adoption by the WG, but no adoption call has been started.", "name": "Candidate for WG Adoption", "next_states": [ 35 @@ -2152,7 +2152,7 @@ }, { "fields": { - "desc": "In some areas, it can be desirable to wait for multiple interoperable implementations before progressing a draft to be an RFC, and in some WGs this is required. This state should be entered after WG Last Call has completed.", + "desc": "The progression of this Working Group (WG) document towards publication is paused as it awaits implementation. The process governing the approach to implementations is WG-specific.", "name": "Waiting for Implementation", "next_states": [], "order": 8, @@ -2165,7 +2165,7 @@ }, { "fields": { - "desc": "Held by WG, see document history for details.", + "desc": "Held by Working Group (WG) chairs for administrative reasons. See document history for details.", "name": "Held by WG", "next_states": [], "order": 9, @@ -2630,6 +2630,58 @@ "model": "doc.state", "pk": 181 }, + { + "fields": { + "desc": "The statement has been marked historic", + "name": "Historic", + "next_states": [], + "order": 0, + "slug": "historic", + "type": "statement", + "used": false + }, + "model": "doc.state", + "pk": 182 + }, + { + "fields": { + "desc": "The statement is no longer active", + "name": "Inactive", + "next_states": [], + "order": 0, + "slug": "inactive", + "type": "statement", + "used": true + }, + "model": "doc.state", + "pk": 183 + }, + { + "fields": { + "desc": "", + "name": "In Progress", + "next_states": [], + "order": 0, + "slug": "in_progress", + "type": "draft-rfceditor", + "used": true + }, + "model": "doc.state", + "pk": 216 + }, + { + "fields": { + "desc": "", + "name": "Blocked", + "next_states": [], + "order": 0, + "slug": "blocked", + "type": "draft-rfceditor", + "used": true + }, + "model": "doc.state", + "pk": 217 + }, { "fields": { "label": "State" @@ -2872,7 +2924,9 @@ "fields": { "about_page": "ietf.group.views.group_about", "acts_like_wg": false, - "admin_roles": "[\n \"chair\"\n]", + "admin_roles": [ + "chair" + ], "agenda_filter_type": "special", "agenda_type": "ietf", "create_wiki": true, @@ -2880,10 +2934,24 @@ "customize_workflow": false, "default_parent": "", "default_tab": "ietf.group.views.group_about", - "default_used_roles": "[\n \"matman\",\n \"ad\",\n \"chair\",\n \"lead\",\n \"delegate\"\n]", - "docman_roles": "[\n \"chair\"\n]", - "groupman_authroles": "[\n \"Secretariat\"\n]", - "groupman_roles": "[\n \"chair\",\n \"lead\",\n \"delegate\"\n]", + "default_used_roles": [ + "matman", + "ad", + "chair", + "lead", + "delegate" + ], + "docman_roles": [ + "chair" + ], + "groupman_authroles": [ + "Secretariat" + ], + "groupman_roles": [ + "chair", + "lead", + "delegate" + ], "has_chartering_process": false, "has_default_chat": true, "has_documents": false, @@ -2893,15 +2961,29 @@ "has_reviews": false, "has_session_materials": true, "is_schedulable": true, - "material_types": "[\n \"slides\"\n]", - "matman_roles": "[\n \"chair\",\n \"lead\",\n \"delegate\",\n \"matman\"\n]", + "material_types": [ + "slides" + ], + "matman_roles": [ + "chair", + "lead", + "delegate", + "matman" + ], "need_parent": false, "parent_types": [ "ietf" ], "req_subm_approval": true, - "role_order": "[\n \"chair\",\n \"lead\",\n \"delegate\",\n \"matman\"\n]", - "session_purposes": "[\n \"presentation\"\n]", + "role_order": [ + "chair", + "lead", + "delegate", + "matman" + ], + "session_purposes": [ + "presentation" + ], "show_on_agenda": true }, "model": "group.groupfeatures", @@ -2911,7 +2993,9 @@ "fields": { "about_page": "ietf.group.views.group_about", "acts_like_wg": false, - "admin_roles": "[\n \"chair\"\n]", + "admin_roles": [ + "chair" + ], "agenda_filter_type": "heading", "agenda_type": "ietf", "create_wiki": false, @@ -2919,10 +3003,19 @@ "customize_workflow": false, "default_parent": "", "default_tab": "ietf.group.views.group_about", - "default_used_roles": "[\n \"member\",\n \"chair\"\n]", - "docman_roles": "[\n \"chair\"\n]", - "groupman_authroles": "[\n \"Secretariat\"\n]", - "groupman_roles": "[\n \"chair\"\n]", + "default_used_roles": [ + "member", + "chair" + ], + "docman_roles": [ + "chair" + ], + "groupman_authroles": [ + "Secretariat" + ], + "groupman_roles": [ + "chair" + ], "has_chartering_process": false, "has_default_chat": false, "has_documents": false, @@ -2932,13 +3025,22 @@ "has_reviews": false, "has_session_materials": false, "is_schedulable": true, - "material_types": "[\n \"slides\"\n]", - "matman_roles": "[\n \"chair\"\n]", + "material_types": [ + "slides" + ], + "matman_roles": [ + "chair" + ], "need_parent": false, "parent_types": [], "req_subm_approval": false, - "role_order": "[\n \"chair\"\n]", - "session_purposes": "[\n \"closed_meeting\",\n \"officehours\"\n]", + "role_order": [ + "chair" + ], + "session_purposes": [ + "closed_meeting", + "officehours" + ], "show_on_agenda": true }, "model": "group.groupfeatures", @@ -2948,7 +3050,9 @@ "fields": { "about_page": "ietf.group.views.group_about", "acts_like_wg": true, - "admin_roles": "[\n \"chair\"\n]", + "admin_roles": [ + "chair" + ], "agenda_filter_type": "normal", "agenda_type": "ietf", "create_wiki": true, @@ -2956,10 +3060,26 @@ "customize_workflow": false, "default_parent": "", "default_tab": "ietf.group.views.group_about", - "default_used_roles": "[\n \"ad\",\n \"chair\",\n \"secr\",\n \"delegate\"\n]", - "docman_roles": "[\n \"chair\",\n \"delegate\",\n \"secr\"\n]", - "groupman_authroles": "[\n \"Secretariat\",\n \"Area Director\"\n]", - "groupman_roles": "[\n \"ad\",\n \"chair\",\n \"delegate\"\n]", + "default_used_roles": [ + "ad", + "chair", + "secr", + "delegate" + ], + "docman_roles": [ + "chair", + "delegate", + "secr" + ], + "groupman_authroles": [ + "Secretariat", + "Area Director" + ], + "groupman_roles": [ + "ad", + "chair", + "delegate" + ], "has_chartering_process": false, "has_default_chat": false, "has_documents": true, @@ -2969,16 +3089,28 @@ "has_reviews": false, "has_session_materials": true, "is_schedulable": true, - "material_types": "[\n \"slides\"\n]", - "matman_roles": "[\n \"ad\",\n \"chair\",\n \"delegate\",\n \"secr\"\n]", + "material_types": [ + "slides" + ], + "matman_roles": [ + "ad", + "chair", + "delegate", + "secr" + ], "need_parent": false, "parent_types": [ "area", "ietf" ], "req_subm_approval": true, - "role_order": "[\n \"chair\",\n \"secr\"\n]", - "session_purposes": "[\n \"regular\"\n]", + "role_order": [ + "chair", + "secr" + ], + "session_purposes": [ + "regular" + ], "show_on_agenda": true }, "model": "group.groupfeatures", @@ -2988,7 +3120,9 @@ "fields": { "about_page": "ietf.group.views.group_about", "acts_like_wg": false, - "admin_roles": "[\n \"ad\"\n]", + "admin_roles": [ + "ad" + ], "agenda_filter_type": "heading", "agenda_type": "ietf", "create_wiki": true, @@ -2996,10 +3130,22 @@ "customize_workflow": false, "default_parent": "", "default_tab": "ietf.group.views.group_about", - "default_used_roles": "[\n \"ad\",\n \"liaison_contact\",\n \"liaison_cc_contact\"\n]", - "docman_roles": "[\n \"ad\",\n \"delegate\",\n \"secr\"\n]", - "groupman_authroles": "[\n \"Secretariat\"\n]", - "groupman_roles": "[\n \"ad\"\n]", + "default_used_roles": [ + "ad", + "liaison_contact", + "liaison_cc_contact" + ], + "docman_roles": [ + "ad", + "delegate", + "secr" + ], + "groupman_authroles": [ + "Secretariat" + ], + "groupman_roles": [ + "ad" + ], "has_chartering_process": false, "has_default_chat": false, "has_documents": false, @@ -3009,15 +3155,27 @@ "has_reviews": false, "has_session_materials": false, "is_schedulable": false, - "material_types": "[\n \"slides\"\n]", - "matman_roles": "[\n \"ad\",\n \"chair\",\n \"delegate\",\n \"secr\"\n]", + "material_types": [ + "slides" + ], + "matman_roles": [ + "ad", + "chair", + "delegate", + "secr" + ], "need_parent": true, "parent_types": [ "ietf" ], "req_subm_approval": true, - "role_order": "[\n \"chair\",\n \"secr\"\n]", - "session_purposes": "[\n \"regular\"\n]", + "role_order": [ + "chair", + "secr" + ], + "session_purposes": [ + "regular" + ], "show_on_agenda": false }, "model": "group.groupfeatures", @@ -3027,7 +3185,10 @@ "fields": { "about_page": "ietf.group.views.group_about", "acts_like_wg": false, - "admin_roles": "[\n \"chair\",\n \"secr\"\n]", + "admin_roles": [ + "chair", + "secr" + ], "agenda_filter_type": "special", "agenda_type": "ad", "create_wiki": true, @@ -3035,10 +3196,25 @@ "customize_workflow": false, "default_parent": "", "default_tab": "ietf.group.views.group_about", - "default_used_roles": "[\n \"ad\",\n \"chair\",\n \"reviewer\",\n \"secr\",\n \"delegate\"\n]", - "docman_roles": "[\n \"chair\"\n]", - "groupman_authroles": "[\n \"Secretariat\"\n]", - "groupman_roles": "[\n \"ad\",\n \"secr\",\n \"delegate\",\n \"chair\"\n]", + "default_used_roles": [ + "ad", + "chair", + "reviewer", + "secr", + "delegate" + ], + "docman_roles": [ + "chair" + ], + "groupman_authroles": [ + "Secretariat" + ], + "groupman_roles": [ + "ad", + "secr", + "delegate", + "chair" + ], "has_chartering_process": false, "has_default_chat": false, "has_documents": false, @@ -3048,15 +3224,31 @@ "has_reviews": false, "has_session_materials": true, "is_schedulable": false, - "material_types": "[\n \"slides\"\n]", - "matman_roles": "[\n \"ad\",\n \"chair\",\n \"delegate\",\n \"secr\"\n]", + "material_types": [ + "slides" + ], + "matman_roles": [ + "ad", + "chair", + "delegate", + "secr" + ], "need_parent": true, "parent_types": [ "area" ], "req_subm_approval": true, - "role_order": "[\n \"chair\",\n \"secr\"\n]", - "session_purposes": "[\n \"open_meeting\",\n \"presentation\",\n \"regular\",\n \"social\",\n \"tutorial\"\n]", + "role_order": [ + "chair", + "secr" + ], + "session_purposes": [ + "open_meeting", + "presentation", + "regular", + "social", + "tutorial" + ], "show_on_agenda": false }, "model": "group.groupfeatures", @@ -3066,7 +3258,9 @@ "fields": { "about_page": "ietf.group.views.group_about", "acts_like_wg": false, - "admin_roles": "[\n \"chair\"\n]", + "admin_roles": [ + "chair" + ], "agenda_filter_type": "normal", "agenda_type": "ietf", "create_wiki": false, @@ -3074,10 +3268,19 @@ "customize_workflow": false, "default_parent": "", "default_tab": "ietf.group.views.group_about", - "default_used_roles": "[\n \"chair\",\n \"member\"\n]", - "docman_roles": "[\n \"chair\"\n]", - "groupman_authroles": "[\n \"Secretariat\"\n]", - "groupman_roles": "[\n \"chair\"\n]", + "default_used_roles": [ + "chair", + "member" + ], + "docman_roles": [ + "chair" + ], + "groupman_authroles": [ + "Secretariat" + ], + "groupman_roles": [ + "chair" + ], "has_chartering_process": false, "has_default_chat": true, "has_documents": false, @@ -3087,13 +3290,23 @@ "has_reviews": false, "has_session_materials": true, "is_schedulable": true, - "material_types": "[\n \"slides\"\n]", - "matman_roles": "[\n \"chair\"\n]", + "material_types": [ + "slides" + ], + "matman_roles": [ + "chair" + ], "need_parent": false, "parent_types": [], "req_subm_approval": false, - "role_order": "[\n \"chair\",\n \"member\"\n]", - "session_purposes": "[\n \"officehours\",\n \"regular\"\n]", + "role_order": [ + "chair", + "member" + ], + "session_purposes": [ + "officehours", + "regular" + ], "show_on_agenda": true }, "model": "group.groupfeatures", @@ -3103,7 +3316,9 @@ "fields": { "about_page": "ietf.group.views.group_about", "acts_like_wg": true, - "admin_roles": "[\n \"chair\"\n]", + "admin_roles": [ + "chair" + ], "agenda_filter_type": "normal", "agenda_type": "ietf", "create_wiki": false, @@ -3111,10 +3326,18 @@ "customize_workflow": true, "default_parent": "", "default_tab": "ietf.group.views.group_documents", - "default_used_roles": "[\n \"chair\"\n]", - "docman_roles": "[\n \"chair\"\n]", - "groupman_authroles": "[\n \"Secretariat\"\n]", - "groupman_roles": "[\n \"chair\"\n]", + "default_used_roles": [ + "chair" + ], + "docman_roles": [ + "chair" + ], + "groupman_authroles": [ + "Secretariat" + ], + "groupman_roles": [ + "chair" + ], "has_chartering_process": false, "has_default_chat": true, "has_documents": true, @@ -3124,15 +3347,23 @@ "has_reviews": false, "has_session_materials": true, "is_schedulable": true, - "material_types": "[\n \"slides\"\n]", - "matman_roles": "[\n \"chair\"\n]", + "material_types": [ + "slides" + ], + "matman_roles": [ + "chair" + ], "need_parent": false, "parent_types": [ "rfcedtyp" ], "req_subm_approval": true, - "role_order": "[\n \"chair\"\n]", - "session_purposes": "[\n \"regular\"\n]", + "role_order": [ + "chair" + ], + "session_purposes": [ + "regular" + ], "show_on_agenda": true }, "model": "group.groupfeatures", @@ -3142,7 +3373,9 @@ "fields": { "about_page": "ietf.group.views.group_about", "acts_like_wg": false, - "admin_roles": "[\n \"chair\"\n]", + "admin_roles": [ + "chair" + ], "agenda_filter_type": "normal", "agenda_type": "ietf", "create_wiki": false, @@ -3150,10 +3383,16 @@ "customize_workflow": false, "default_parent": "", "default_tab": "ietf.group.views.group_about", - "default_used_roles": "[\n \"chair\"\n]", - "docman_roles": "[\n \"chair\"\n]", - "groupman_authroles": "[\n \"Secretariat\"\n]", - "groupman_roles": "[]", + "default_used_roles": [ + "chair" + ], + "docman_roles": [ + "chair" + ], + "groupman_authroles": [ + "Secretariat" + ], + "groupman_roles": [], "has_chartering_process": false, "has_default_chat": false, "has_documents": false, @@ -3163,15 +3402,26 @@ "has_reviews": false, "has_session_materials": false, "is_schedulable": false, - "material_types": "[\n \"slides\"\n]", - "matman_roles": "[\n \"chair\",\n \"delegate\"\n]", + "material_types": [ + "slides" + ], + "matman_roles": [ + "chair", + "delegate" + ], "need_parent": false, "parent_types": [ "ietf" ], "req_subm_approval": true, - "role_order": "[\n \"chair\",\n \"secr\"\n]", - "session_purposes": "[\n \"closed_meeting\",\n \"regular\"\n]", + "role_order": [ + "chair", + "secr" + ], + "session_purposes": [ + "closed_meeting", + "regular" + ], "show_on_agenda": true }, "model": "group.groupfeatures", @@ -3181,7 +3431,9 @@ "fields": { "about_page": "ietf.group.views.group_about", "acts_like_wg": false, - "admin_roles": "[\n \"lead\"\n]", + "admin_roles": [ + "lead" + ], "agenda_filter_type": "none", "agenda_type": "ad", "create_wiki": false, @@ -3189,10 +3441,27 @@ "customize_workflow": false, "default_parent": "", "default_tab": "ietf.group.views.group_about", - "default_used_roles": "[\n \"member\",\n \"chair\",\n \"lead\",\n \"delegate\"\n]", - "docman_roles": "[\n \"lead\",\n \"chair\",\n \"secr\"\n]", - "groupman_authroles": "[\n \"Secretariat\",\n \"IAB\"\n]", - "groupman_roles": "[\n \"lead\",\n \"chair\",\n \"secr\",\n \"delegate\"\n]", + "default_used_roles": [ + "member", + "chair", + "lead", + "delegate" + ], + "docman_roles": [ + "lead", + "chair", + "secr" + ], + "groupman_authroles": [ + "Secretariat", + "IAB" + ], + "groupman_roles": [ + "lead", + "chair", + "secr", + "delegate" + ], "has_chartering_process": false, "has_default_chat": false, "has_documents": true, @@ -3202,15 +3471,29 @@ "has_reviews": false, "has_session_materials": false, "is_schedulable": false, - "material_types": "[\n \"slides\"\n]", - "matman_roles": "[\n \"lead\",\n \"chair\",\n \"secr\"\n]", + "material_types": [ + "slides" + ], + "matman_roles": [ + "lead", + "chair", + "secr" + ], "need_parent": false, "parent_types": [ "ietf" ], "req_subm_approval": false, - "role_order": "[\n \"lead\",\n \"chair\",\n \"secr\"\n]", - "session_purposes": "[\n \"closed_meeting\",\n \"officehours\",\n \"open_meeting\"\n]", + "role_order": [ + "lead", + "chair", + "secr" + ], + "session_purposes": [ + "closed_meeting", + "officehours", + "open_meeting" + ], "show_on_agenda": false }, "model": "group.groupfeatures", @@ -3220,7 +3503,9 @@ "fields": { "about_page": "ietf.group.views.group_about", "acts_like_wg": false, - "admin_roles": "[\n \"chair\"\n]", + "admin_roles": [ + "chair" + ], "agenda_filter_type": "none", "agenda_type": "ietf", "create_wiki": false, @@ -3228,10 +3513,20 @@ "customize_workflow": false, "default_parent": "iab", "default_tab": "ietf.group.views.group_about", - "default_used_roles": "[]", - "docman_roles": "[\n \"ad\",\n \"chair\",\n \"delegate\",\n \"secr\"\n]", - "groupman_authroles": "[\n \"Secretariat\"\n]", - "groupman_roles": "[\n \"ad\",\n \"chair\"\n]", + "default_used_roles": [], + "docman_roles": [ + "ad", + "chair", + "delegate", + "secr" + ], + "groupman_authroles": [ + "Secretariat" + ], + "groupman_roles": [ + "ad", + "chair" + ], "has_chartering_process": false, "has_default_chat": true, "has_documents": true, @@ -3241,15 +3536,26 @@ "has_reviews": false, "has_session_materials": true, "is_schedulable": false, - "material_types": "[\n \"slides\"\n]", - "matman_roles": "[\n \"ad\",\n \"chair\",\n \"delegate\",\n \"secr\"\n]", + "material_types": [ + "slides" + ], + "matman_roles": [ + "ad", + "chair", + "delegate", + "secr" + ], "need_parent": true, "parent_types": [ "ietf" ], "req_subm_approval": false, - "role_order": "[\n \"chair\",\n \"secr\",\n \"member\"\n]", - "session_purposes": "\"[\\\"regular\\\"]\"", + "role_order": [ + "chair", + "secr", + "member" + ], + "session_purposes": "[\"regular\"]", "show_on_agenda": false }, "model": "group.groupfeatures", @@ -3259,7 +3565,9 @@ "fields": { "about_page": "ietf.group.views.group_about", "acts_like_wg": false, - "admin_roles": "[\n \"chair\"\n]", + "admin_roles": [ + "chair" + ], "agenda_filter_type": "none", "agenda_type": "ietf", "create_wiki": false, @@ -3267,10 +3575,18 @@ "customize_workflow": false, "default_parent": "", "default_tab": "ietf.group.views.group_about", - "default_used_roles": "[\n \"auth\"\n]", - "docman_roles": "[\n \"chair\"\n]", - "groupman_authroles": "[\n \"Secretariat\"\n]", - "groupman_roles": "[\n \"chair\"\n]", + "default_used_roles": [ + "auth" + ], + "docman_roles": [ + "chair" + ], + "groupman_authroles": [ + "Secretariat" + ], + "groupman_roles": [ + "chair" + ], "has_chartering_process": false, "has_default_chat": false, "has_documents": false, @@ -3280,13 +3596,21 @@ "has_reviews": false, "has_session_materials": false, "is_schedulable": false, - "material_types": "[\n \"slides\"\n]", - "matman_roles": "[\n \"chair\"\n]", + "material_types": [ + "slides" + ], + "matman_roles": [ + "chair" + ], "need_parent": false, "parent_types": [], "req_subm_approval": false, - "role_order": "[\n \"chair\"\n]", - "session_purposes": "[\n \"officehours\"\n]", + "role_order": [ + "chair" + ], + "session_purposes": [ + "officehours" + ], "show_on_agenda": false }, "model": "group.groupfeatures", @@ -3296,7 +3620,9 @@ "fields": { "about_page": "ietf.group.views.group_about", "acts_like_wg": false, - "admin_roles": "[\n \"chair\"\n]", + "admin_roles": [ + "chair" + ], "agenda_filter_type": "none", "agenda_type": "ad", "create_wiki": false, @@ -3304,10 +3630,19 @@ "customize_workflow": false, "default_parent": "", "default_tab": "ietf.group.views.group_about", - "default_used_roles": "[\n \"delegate\"\n]", - "docman_roles": "[\n \"chair\"\n]", - "groupman_authroles": "[\n \"Secretariat\"\n]", - "groupman_roles": "[\n \"chair\",\n \"delegate\"\n]", + "default_used_roles": [ + "delegate" + ], + "docman_roles": [ + "chair" + ], + "groupman_authroles": [ + "Secretariat" + ], + "groupman_roles": [ + "chair", + "delegate" + ], "has_chartering_process": false, "has_default_chat": false, "has_documents": false, @@ -3317,13 +3652,24 @@ "has_reviews": false, "has_session_materials": false, "is_schedulable": false, - "material_types": "\"[]\"", - "matman_roles": "[\n \"chair\",\n \"delegate\",\n \"member\"\n]", + "material_types": "[]", + "matman_roles": [ + "chair", + "delegate", + "member" + ], "need_parent": false, "parent_types": [], "req_subm_approval": true, - "role_order": "[\n \"chair\",\n \"delegate\",\n \"member\"\n]", - "session_purposes": "[\n \"closed_meeting\",\n \"open_meeting\"\n]", + "role_order": [ + "chair", + "delegate", + "member" + ], + "session_purposes": [ + "closed_meeting", + "open_meeting" + ], "show_on_agenda": false }, "model": "group.groupfeatures", @@ -3333,7 +3679,10 @@ "fields": { "about_page": "ietf.group.views.group_about", "acts_like_wg": false, - "admin_roles": "[\n \"chair\",\n \"lead\"\n]", + "admin_roles": [ + "chair", + "lead" + ], "agenda_filter_type": "heading", "agenda_type": "ietf", "create_wiki": false, @@ -3341,10 +3690,26 @@ "customize_workflow": false, "default_parent": "", "default_tab": "ietf.group.views.group_about", - "default_used_roles": "[\n \"ad\",\n \"member\",\n \"comdir\",\n \"delegate\",\n \"execdir\",\n \"recman\",\n \"secr\",\n \"trac-editor\",\n \"trac-admin\",\n \"chair\"\n]", - "docman_roles": "[\n \"chair\"\n]", - "groupman_authroles": "[\n \"Secretariat\"\n]", - "groupman_roles": "[\n \"chair\",\n \"delegate\"\n]", + "default_used_roles": [ + "ad", + "member", + "comdir", + "delegate", + "execdir", + "recman", + "secr", + "chair" + ], + "docman_roles": [ + "chair" + ], + "groupman_authroles": [ + "Secretariat" + ], + "groupman_roles": [ + "chair", + "delegate" + ], "has_chartering_process": false, "has_default_chat": false, "has_documents": false, @@ -3354,15 +3719,29 @@ "has_reviews": false, "has_session_materials": true, "is_schedulable": false, - "material_types": "[\n \"slides\"\n]", - "matman_roles": "[\n \"chair\",\n \"delegate\"\n]", + "material_types": [ + "slides" + ], + "matman_roles": [ + "chair", + "delegate" + ], "need_parent": false, "parent_types": [ "ietf" ], "req_subm_approval": true, - "role_order": "[\n \"chair\",\n \"secr\"\n]", - "session_purposes": "[\n \"admin\",\n \"plenary\",\n \"presentation\",\n \"social\",\n \"officehours\"\n]", + "role_order": [ + "chair", + "secr" + ], + "session_purposes": [ + "admin", + "plenary", + "presentation", + "social", + "officehours" + ], "show_on_agenda": false }, "model": "group.groupfeatures", @@ -3372,7 +3751,9 @@ "fields": { "about_page": "ietf.group.views.group_about", "acts_like_wg": false, - "admin_roles": "[\n \"chair\"\n]", + "admin_roles": [ + "chair" + ], "agenda_filter_type": "none", "agenda_type": "ad", "create_wiki": false, @@ -3380,10 +3761,16 @@ "customize_workflow": false, "default_parent": "", "default_tab": "ietf.group.views.group_about", - "default_used_roles": "[\n \"ad\"\n]", - "docman_roles": "[\n \"auth\"\n]", - "groupman_authroles": "[\n \"Secretariat\"\n]", - "groupman_roles": "[]", + "default_used_roles": [ + "ad" + ], + "docman_roles": [ + "auth" + ], + "groupman_authroles": [ + "Secretariat" + ], + "groupman_roles": [], "has_chartering_process": false, "has_default_chat": false, "has_documents": false, @@ -3393,15 +3780,20 @@ "has_reviews": false, "has_session_materials": false, "is_schedulable": false, - "material_types": "[\n \"slides\"\n]", - "matman_roles": "[]", + "material_types": [ + "slides" + ], + "matman_roles": [], "need_parent": true, "parent_types": [ "area" ], "req_subm_approval": false, - "role_order": "[\n \"chair\",\n \"secr\"\n]", - "session_purposes": "[]", + "role_order": [ + "chair", + "secr" + ], + "session_purposes": [], "show_on_agenda": false }, "model": "group.groupfeatures", @@ -3411,7 +3803,9 @@ "fields": { "about_page": "ietf.group.views.group_about", "acts_like_wg": false, - "admin_roles": "[\n \"chair\"\n]", + "admin_roles": [ + "chair" + ], "agenda_filter_type": "heading", "agenda_type": "ietf", "create_wiki": false, @@ -3419,10 +3813,20 @@ "customize_workflow": false, "default_parent": "", "default_tab": "ietf.group.views.group_about", - "default_used_roles": "[\n \"member\",\n \"atlarge\",\n \"chair\",\n \"delegate\"\n]", - "docman_roles": "[]", - "groupman_authroles": "[\n \"Secretariat\"\n]", - "groupman_roles": "[\n \"chair\",\n \"delegate\"\n]", + "default_used_roles": [ + "member", + "atlarge", + "chair", + "delegate" + ], + "docman_roles": [], + "groupman_authroles": [ + "Secretariat" + ], + "groupman_roles": [ + "chair", + "delegate" + ], "has_chartering_process": false, "has_default_chat": false, "has_documents": false, @@ -3432,15 +3836,24 @@ "has_reviews": false, "has_session_materials": false, "is_schedulable": false, - "material_types": "[\n \"slides\"\n]", - "matman_roles": "[\n \"chair\",\n \"delegate\",\n \"secr\"\n]", + "material_types": [ + "slides" + ], + "matman_roles": [ + "chair", + "delegate", + "secr" + ], "need_parent": false, "parent_types": [ "irtf" ], "req_subm_approval": true, - "role_order": "[\n \"chair\",\n \"secr\"\n]", - "session_purposes": "[]", + "role_order": [ + "chair", + "secr" + ], + "session_purposes": [], "show_on_agenda": false }, "model": "group.groupfeatures", @@ -3450,7 +3863,10 @@ "fields": { "about_page": "ietf.group.views.group_about", "acts_like_wg": false, - "admin_roles": "[\n \"chair\",\n \"lead\"\n]", + "admin_roles": [ + "chair", + "lead" + ], "agenda_filter_type": "heading", "agenda_type": "ietf", "create_wiki": false, @@ -3458,10 +3874,20 @@ "customize_workflow": false, "default_parent": "", "default_tab": "ietf.group.views.group_about", - "default_used_roles": "[\n \"chair\",\n \"delegate\"\n]", - "docman_roles": "[\n \"chair\"\n]", - "groupman_authroles": "[\n \"Secretariat\"\n]", - "groupman_roles": "[\n \"chair\",\n \"delegate\"\n]", + "default_used_roles": [ + "chair", + "delegate" + ], + "docman_roles": [ + "chair" + ], + "groupman_authroles": [ + "Secretariat" + ], + "groupman_roles": [ + "chair", + "delegate" + ], "has_chartering_process": false, "has_default_chat": false, "has_documents": true, @@ -3471,13 +3897,24 @@ "has_reviews": false, "has_session_materials": false, "is_schedulable": true, - "material_types": "[\n \"slides\"\n]", - "matman_roles": "[\n \"chair\",\n \"delegate\"\n]", + "material_types": [ + "slides" + ], + "matman_roles": [ + "chair", + "delegate" + ], "need_parent": false, "parent_types": [], "req_subm_approval": true, - "role_order": "[\n \"chair\",\n \"delegate\"\n]", - "session_purposes": "[\n \"officehours\",\n \"regular\"\n]", + "role_order": [ + "chair", + "delegate" + ], + "session_purposes": [ + "officehours", + "regular" + ], "show_on_agenda": true }, "model": "group.groupfeatures", @@ -3487,7 +3924,9 @@ "fields": { "about_page": "ietf.group.views.group_about", "acts_like_wg": false, - "admin_roles": "[\n \"chair\"\n]", + "admin_roles": [ + "chair" + ], "agenda_filter_type": "none", "agenda_type": null, "create_wiki": false, @@ -3495,10 +3934,17 @@ "customize_workflow": false, "default_parent": "", "default_tab": "ietf.group.views.group_about", - "default_used_roles": "[\n \"chair\",\n \"ceo\"\n]", - "docman_roles": "[]", - "groupman_authroles": "[\n \"Secretariat\"\n]", - "groupman_roles": "[\n \"chair\"\n]", + "default_used_roles": [ + "chair", + "ceo" + ], + "docman_roles": [], + "groupman_authroles": [ + "Secretariat" + ], + "groupman_roles": [ + "chair" + ], "has_chartering_process": false, "has_default_chat": false, "has_documents": false, @@ -3508,15 +3954,27 @@ "has_reviews": false, "has_session_materials": false, "is_schedulable": false, - "material_types": "[\n \"slides\"\n]", - "matman_roles": "[\n \"chair\",\n \"secr\"\n]", + "material_types": [ + "slides" + ], + "matman_roles": [ + "chair", + "secr" + ], "need_parent": false, "parent_types": [ "isoc" ], "req_subm_approval": true, - "role_order": "[\n \"chair\",\n \"secr\"\n]", - "session_purposes": "[\n \"officehours\",\n \"open_meeting\",\n \"presentation\"\n]", + "role_order": [ + "chair", + "secr" + ], + "session_purposes": [ + "officehours", + "open_meeting", + "presentation" + ], "show_on_agenda": false }, "model": "group.groupfeatures", @@ -3526,7 +3984,10 @@ "fields": { "about_page": "ietf.group.views.group_about", "acts_like_wg": false, - "admin_roles": "[\n \"chair\",\n \"advisor\"\n]", + "admin_roles": [ + "chair", + "advisor" + ], "agenda_filter_type": "none", "agenda_type": "side", "create_wiki": true, @@ -3534,10 +3995,23 @@ "customize_workflow": false, "default_parent": "", "default_tab": "ietf.group.views.group_about", - "default_used_roles": "[\n \"member\",\n \"advisor\",\n \"liaison\",\n \"chair\",\n \"techadv\"\n]", - "docman_roles": "[\n \"chair\"\n]", - "groupman_authroles": "[\n \"Secretariat\"\n]", - "groupman_roles": "[\n \"chair\",\n \"advisor\"\n]", + "default_used_roles": [ + "member", + "advisor", + "liaison", + "chair", + "techadv" + ], + "docman_roles": [ + "chair" + ], + "groupman_authroles": [ + "Secretariat" + ], + "groupman_roles": [ + "chair", + "advisor" + ], "has_chartering_process": false, "has_default_chat": false, "has_documents": false, @@ -3547,15 +4021,26 @@ "has_reviews": false, "has_session_materials": false, "is_schedulable": false, - "material_types": "[\n \"slides\"\n]", - "matman_roles": "[\n \"chair\"\n]", + "material_types": [ + "slides" + ], + "matman_roles": [ + "chair" + ], "need_parent": false, "parent_types": [ "area" ], "req_subm_approval": true, - "role_order": "[\n \"chair\",\n \"member\",\n \"advisor\"\n]", - "session_purposes": "[\n \"closed_meeting\",\n \"officehours\"\n]", + "role_order": [ + "chair", + "member", + "advisor" + ], + "session_purposes": [ + "closed_meeting", + "officehours" + ], "show_on_agenda": false }, "model": "group.groupfeatures", @@ -3565,7 +4050,9 @@ "fields": { "about_page": "ietf.group.views.group_about", "acts_like_wg": false, - "admin_roles": "[\n \"lead\"\n]", + "admin_roles": [ + "lead" + ], "agenda_filter_type": "normal", "agenda_type": "ad", "create_wiki": false, @@ -3573,10 +4060,27 @@ "customize_workflow": false, "default_parent": "", "default_tab": "ietf.group.views.group_about", - "default_used_roles": "[\n \"member\",\n \"chair\",\n \"lead\",\n \"delegate\"\n]", - "docman_roles": "[\n \"lead\",\n \"chair\",\n \"secr\"\n]", - "groupman_authroles": "[\n \"Secretariat\",\n \"IAB\"\n]", - "groupman_roles": "[\n \"lead\",\n \"chair\",\n \"secr\",\n \"delegate\"\n]", + "default_used_roles": [ + "member", + "chair", + "lead", + "delegate" + ], + "docman_roles": [ + "lead", + "chair", + "secr" + ], + "groupman_authroles": [ + "Secretariat", + "IAB" + ], + "groupman_roles": [ + "lead", + "chair", + "secr", + "delegate" + ], "has_chartering_process": false, "has_default_chat": false, "has_documents": true, @@ -3586,15 +4090,28 @@ "has_reviews": false, "has_session_materials": true, "is_schedulable": false, - "material_types": "[\n \"slides\"\n]", - "matman_roles": "[\n \"lead\",\n \"chair\",\n \"secr\"\n]", + "material_types": [ + "slides" + ], + "matman_roles": [ + "lead", + "chair", + "secr" + ], "need_parent": false, "parent_types": [ "ietf" ], "req_subm_approval": false, - "role_order": "[\n \"lead\",\n \"chair\",\n \"secr\"\n]", - "session_purposes": "[\n \"regular\",\n \"tutorial\"\n]", + "role_order": [ + "lead", + "chair", + "secr" + ], + "session_purposes": [ + "regular", + "tutorial" + ], "show_on_agenda": false }, "model": "group.groupfeatures", @@ -3604,7 +4121,9 @@ "fields": { "about_page": "ietf.group.views.group_about", "acts_like_wg": true, - "admin_roles": "[\n \"chair\"\n]", + "admin_roles": [ + "chair" + ], "agenda_filter_type": "normal", "agenda_type": "ietf", "create_wiki": true, @@ -3612,10 +4131,24 @@ "customize_workflow": false, "default_parent": "", "default_tab": "ietf.group.views.group_about", - "default_used_roles": "[\n \"chair\",\n \"secr\",\n \"delegate\"\n]", - "docman_roles": "[\n \"chair\",\n \"delegate\",\n \"secr\"\n]", - "groupman_authroles": "[\n \"Secretariat\",\n \"IRTF Chair\"\n]", - "groupman_roles": "[\n \"chair\",\n \"delegate\"\n]", + "default_used_roles": [ + "chair", + "secr", + "delegate" + ], + "docman_roles": [ + "chair", + "delegate", + "secr" + ], + "groupman_authroles": [ + "Secretariat", + "IRTF Chair" + ], + "groupman_roles": [ + "chair", + "delegate" + ], "has_chartering_process": false, "has_default_chat": false, "has_documents": true, @@ -3625,15 +4158,26 @@ "has_reviews": false, "has_session_materials": true, "is_schedulable": true, - "material_types": "[\n \"slides\"\n]", - "matman_roles": "[\n \"chair\",\n \"delegate\",\n \"secr\"\n]", + "material_types": [ + "slides" + ], + "matman_roles": [ + "chair", + "delegate", + "secr" + ], "need_parent": false, "parent_types": [ "irtf" ], "req_subm_approval": true, - "role_order": "[\n \"chair\",\n \"secr\"\n]", - "session_purposes": "[\n \"regular\"\n]", + "role_order": [ + "chair", + "secr" + ], + "session_purposes": [ + "regular" + ], "show_on_agenda": true }, "model": "group.groupfeatures", @@ -3643,7 +4187,10 @@ "fields": { "about_page": "ietf.group.views.group_about", "acts_like_wg": false, - "admin_roles": "[\n \"chair\",\n \"secr\"\n]", + "admin_roles": [ + "chair", + "secr" + ], "agenda_filter_type": "normal", "agenda_type": "ietf", "create_wiki": true, @@ -3651,10 +4198,24 @@ "customize_workflow": false, "default_parent": "", "default_tab": "ietf.group.views.review_requests", - "default_used_roles": "[\n \"ad\",\n \"chair\",\n \"reviewer\",\n \"secr\",\n \"delegate\"\n]", - "docman_roles": "[\n \"secr\"\n]", - "groupman_authroles": "[\n \"Secretariat\"\n]", - "groupman_roles": "[\n \"ad\",\n \"secr\",\n \"delegate\"\n]", + "default_used_roles": [ + "ad", + "chair", + "reviewer", + "secr", + "delegate" + ], + "docman_roles": [ + "secr" + ], + "groupman_authroles": [ + "Secretariat" + ], + "groupman_roles": [ + "ad", + "secr", + "delegate" + ], "has_chartering_process": false, "has_default_chat": false, "has_documents": false, @@ -3664,15 +4225,26 @@ "has_reviews": true, "has_session_materials": true, "is_schedulable": false, - "material_types": "[\n \"slides\"\n]", - "matman_roles": "[\n \"ad\",\n \"secr\"\n]", + "material_types": [ + "slides" + ], + "matman_roles": [ + "ad", + "secr" + ], "need_parent": true, "parent_types": [ "area" ], "req_subm_approval": true, - "role_order": "[\n \"chair\",\n \"secr\"\n]", - "session_purposes": "[\n \"open_meeting\",\n \"social\"\n]", + "role_order": [ + "chair", + "secr" + ], + "session_purposes": [ + "open_meeting", + "social" + ], "show_on_agenda": false }, "model": "group.groupfeatures", @@ -3682,7 +4254,9 @@ "fields": { "about_page": "ietf.group.views.group_about", "acts_like_wg": false, - "admin_roles": "[\n \"chair\"\n]", + "admin_roles": [ + "chair" + ], "agenda_filter_type": "normal", "agenda_type": "ietf", "create_wiki": false, @@ -3690,10 +4264,19 @@ "customize_workflow": false, "default_parent": "", "default_tab": "ietf.group.views.group_about", - "default_used_roles": "[\n \"auth\",\n \"chair\"\n]", - "docman_roles": "[\n \"chair\"\n]", - "groupman_authroles": "[\n \"Secretariat\"\n]", - "groupman_roles": "[\n \"chair\"\n]", + "default_used_roles": [ + "auth", + "chair" + ], + "docman_roles": [ + "chair" + ], + "groupman_authroles": [ + "Secretariat" + ], + "groupman_roles": [ + "chair" + ], "has_chartering_process": false, "has_default_chat": false, "has_documents": false, @@ -3703,13 +4286,23 @@ "has_reviews": false, "has_session_materials": false, "is_schedulable": false, - "material_types": "[\n \"slides\"\n]", - "matman_roles": "[\n \"chair\"\n]", + "material_types": [ + "slides" + ], + "matman_roles": [ + "chair" + ], "need_parent": false, "parent_types": [], "req_subm_approval": true, - "role_order": "[\n \"chair\",\n \"secr\"\n]", - "session_purposes": "[\n \"officehours\",\n \"regular\"\n]", + "role_order": [ + "chair", + "secr" + ], + "session_purposes": [ + "officehours", + "regular" + ], "show_on_agenda": false }, "model": "group.groupfeatures", @@ -3719,7 +4312,9 @@ "fields": { "about_page": "ietf.group.views.group_about", "acts_like_wg": true, - "admin_roles": "[\n \"chair\"\n]", + "admin_roles": [ + "chair" + ], "agenda_filter_type": "normal", "agenda_type": "ietf", "create_wiki": true, @@ -3727,10 +4322,25 @@ "customize_workflow": true, "default_parent": "irtf", "default_tab": "ietf.group.views.group_documents", - "default_used_roles": "[\n \"chair\",\n \"techadv\",\n \"secr\",\n \"delegate\"\n]", - "docman_roles": "[\n \"chair\",\n \"delegate\",\n \"secr\"\n]", - "groupman_authroles": "[\n \"Secretariat\",\n \"IRTF Chair\"\n]", - "groupman_roles": "[\n \"chair\",\n \"delegate\"\n]", + "default_used_roles": [ + "chair", + "techadv", + "secr", + "delegate" + ], + "docman_roles": [ + "chair", + "delegate", + "secr" + ], + "groupman_authroles": [ + "Secretariat", + "IRTF Chair" + ], + "groupman_roles": [ + "chair", + "delegate" + ], "has_chartering_process": true, "has_default_chat": true, "has_documents": true, @@ -3740,15 +4350,27 @@ "has_reviews": false, "has_session_materials": true, "is_schedulable": true, - "material_types": "[\n \"slides\"\n]", - "matman_roles": "[\n \"chair\",\n \"delegate\",\n \"secr\"\n]", + "material_types": [ + "slides" + ], + "matman_roles": [ + "chair", + "delegate", + "secr" + ], "need_parent": true, "parent_types": [ "irtf" ], "req_subm_approval": true, - "role_order": "[\n \"chair\",\n \"delegate\",\n \"secr\"\n]", - "session_purposes": "[\n \"regular\"\n]", + "role_order": [ + "chair", + "delegate", + "secr" + ], + "session_purposes": [ + "regular" + ], "show_on_agenda": true }, "model": "group.groupfeatures", @@ -3758,7 +4380,9 @@ "fields": { "about_page": "ietf.group.views.group_about", "acts_like_wg": false, - "admin_roles": "[\n \"chair\"\n]", + "admin_roles": [ + "chair" + ], "agenda_filter_type": "none", "agenda_type": null, "create_wiki": false, @@ -3766,10 +4390,23 @@ "customize_workflow": false, "default_parent": "", "default_tab": "ietf.group.views.group_about", - "default_used_roles": "[\n \"liaiman\",\n \"ceo\",\n \"coord\",\n \"auth\",\n \"chair\",\n \"liaison_contact\",\n \"liaison_cc_contact\"\n]", - "docman_roles": "[\n \"liaiman\",\n \"matman\"\n]", - "groupman_authroles": "[\n \"Secretariat\"\n]", - "groupman_roles": "[]", + "default_used_roles": [ + "liaiman", + "ceo", + "coord", + "auth", + "chair", + "liaison_contact", + "liaison_cc_contact" + ], + "docman_roles": [ + "liaiman", + "matman" + ], + "groupman_authroles": [ + "Secretariat" + ], + "groupman_roles": [], "has_chartering_process": false, "has_default_chat": false, "has_documents": false, @@ -3779,16 +4416,20 @@ "has_reviews": false, "has_session_materials": false, "is_schedulable": false, - "material_types": "[\n \"slides\"\n]", - "matman_roles": "[]", + "material_types": [ + "slides" + ], + "matman_roles": [], "need_parent": false, "parent_types": [ "area", "sdo" ], "req_subm_approval": true, - "role_order": "[\n \"liaiman\"\n]", - "session_purposes": "[]", + "role_order": [ + "liaiman" + ], + "session_purposes": [], "show_on_agenda": false }, "model": "group.groupfeatures", @@ -3798,7 +4439,9 @@ "fields": { "about_page": "ietf.group.views.group_about", "acts_like_wg": false, - "admin_roles": "[\n \"chair\"\n]", + "admin_roles": [ + "chair" + ], "agenda_filter_type": "special", "agenda_type": "ietf", "create_wiki": true, @@ -3806,10 +4449,28 @@ "customize_workflow": false, "default_parent": "", "default_tab": "ietf.group.views.group_about", - "default_used_roles": "[\n \"ad\",\n \"member\",\n \"delegate\",\n \"secr\",\n \"liaison\",\n \"atlarge\",\n \"chair\",\n \"matman\",\n \"techadv\"\n]", - "docman_roles": "[\n \"chair\"\n]", - "groupman_authroles": "[\n \"Secretariat\",\n \"Area Director\"\n]", - "groupman_roles": "[\n \"chair\",\n \"delegate\"\n]", + "default_used_roles": [ + "ad", + "member", + "delegate", + "secr", + "liaison", + "atlarge", + "chair", + "matman", + "techadv" + ], + "docman_roles": [ + "chair" + ], + "groupman_authroles": [ + "Secretariat", + "Area Director" + ], + "groupman_roles": [ + "chair", + "delegate" + ], "has_chartering_process": false, "has_default_chat": false, "has_documents": false, @@ -3819,15 +4480,30 @@ "has_reviews": false, "has_session_materials": true, "is_schedulable": true, - "material_types": "[\n \"slides\"\n]", - "matman_roles": "[\n \"chair\",\n \"matman\"\n]", + "material_types": [ + "slides" + ], + "matman_roles": [ + "chair", + "matman" + ], "need_parent": false, "parent_types": [ "area" ], "req_subm_approval": false, - "role_order": "[\n \"chair\",\n \"member\",\n \"matman\"\n]", - "session_purposes": "[\n \"coding\",\n \"presentation\",\n \"social\",\n \"tutorial\"\n]", + "role_order": [ + "chair", + "member", + "matman" + ], + "session_purposes": [ + "coding", + "open_meeting", + "presentation", + "social", + "tutorial" + ], "show_on_agenda": true }, "model": "group.groupfeatures", @@ -3837,7 +4513,9 @@ "fields": { "about_page": "ietf.group.views.group_about", "acts_like_wg": true, - "admin_roles": "[\n \"chair\"\n]", + "admin_roles": [ + "chair" + ], "agenda_filter_type": "normal", "agenda_type": "ietf", "create_wiki": true, @@ -3845,10 +4523,32 @@ "customize_workflow": true, "default_parent": "", "default_tab": "ietf.group.views.group_documents", - "default_used_roles": "[\n \"ad\",\n \"editor\",\n \"delegate\",\n \"secr\",\n \"chair\",\n \"matman\",\n \"techadv\",\n \"liaison_contact\",\n \"liaison_cc_contact\"\n]", - "docman_roles": "[\n \"chair\",\n \"delegate\",\n \"secr\"\n]", - "groupman_authroles": "[\n \"Secretariat\",\n \"Area Director\"\n]", - "groupman_roles": "[\n \"ad\",\n \"chair\",\n \"delegate\",\n \"secr\"\n]", + "default_used_roles": [ + "ad", + "editor", + "delegate", + "secr", + "chair", + "matman", + "techadv", + "liaison_contact", + "liaison_cc_contact" + ], + "docman_roles": [ + "chair", + "delegate", + "secr" + ], + "groupman_authroles": [ + "Secretariat", + "Area Director" + ], + "groupman_roles": [ + "ad", + "chair", + "delegate", + "secr" + ], "has_chartering_process": true, "has_default_chat": true, "has_documents": true, @@ -3858,15 +4558,28 @@ "has_reviews": false, "has_session_materials": true, "is_schedulable": true, - "material_types": "[\n \"slides\"\n]", - "matman_roles": "[\n \"ad\",\n \"chair\",\n \"delegate\",\n \"secr\"\n]", + "material_types": [ + "slides" + ], + "matman_roles": [ + "ad", + "chair", + "delegate", + "secr" + ], "need_parent": false, "parent_types": [ "area" ], "req_subm_approval": true, - "role_order": "[\n \"chair\",\n \"secr\",\n \"delegate\"\n]", - "session_purposes": "[\n \"regular\"\n]", + "role_order": [ + "chair", + "secr", + "delegate" + ], + "session_purposes": [ + "regular" + ], "show_on_agenda": true }, "model": "group.groupfeatures", @@ -4443,6 +5156,34 @@ "model": "mailtrigger.mailtrigger", "pk": "doc_telechat_details_changed" }, + { + "fields": { + "cc": [], + "desc": "Recipients when a working group call for adoption is issued", + "to": [ + "doc_authors", + "doc_group_chairs", + "doc_group_mail_list", + "doc_shepherd" + ] + }, + "model": "mailtrigger.mailtrigger", + "pk": "doc_wg_call_for_adoption_issued" + }, + { + "fields": { + "cc": [], + "desc": "Recipients when a working group last call is issued", + "to": [ + "doc_authors", + "doc_group_chairs", + "doc_group_mail_list", + "doc_shepherd" + ] + }, + "model": "mailtrigger.mailtrigger", + "pk": "doc_wg_last_call_issued" + }, { "fields": { "cc": [], @@ -4819,13 +5560,30 @@ "liaison_response_contacts", "liaison_technical_contacts" ], - "desc": "Recipient for a message when a new liaison statement is posted", + "desc": "Recipients for a message when a new incoming liaison statement is posted", + "to": [ + "liaison_to_contacts" + ] + }, + "model": "mailtrigger.mailtrigger", + "pk": "liaison_statement_posted_incoming" + }, + { + "fields": { + "cc": [ + "liaison_cc", + "liaison_coordinators", + "liaison_from_contact", + "liaison_response_contacts", + "liaison_technical_contacts" + ], + "desc": "Recipients for a message when a new outgoing liaison statement is posted", "to": [ "liaison_to_contacts" ] }, "model": "mailtrigger.mailtrigger", - "pk": "liaison_statement_posted" + "pk": "liaison_statement_posted_outgoing" }, { "fields": { @@ -5084,6 +5842,20 @@ "model": "mailtrigger.mailtrigger", "pk": "review_completed_artart_telechat" }, + { + "fields": { + "cc": [ + "review_doc_all_parties", + "review_doc_group_mail_list" + ], + "desc": "Recipients when a bgpdir Early review is completed", + "to": [ + "review_team_mail_list" + ] + }, + "model": "mailtrigger.mailtrigger", + "pk": "review_completed_bgpdir_early" + }, { "fields": { "cc": [ @@ -5392,6 +6164,50 @@ "model": "mailtrigger.mailtrigger", "pk": "review_completed_opsdir_telechat" }, + { + "fields": { + "cc": [ + "review_doc_all_parties", + "review_doc_group_mail_list" + ], + "desc": "Recipients when a perfmetrdir Early review is completed", + "to": [ + "review_team_mail_list" + ] + }, + "model": "mailtrigger.mailtrigger", + "pk": "review_completed_perfmetrdir_early" + }, + { + "fields": { + "cc": [ + "ietf_last_call", + "review_doc_all_parties", + "review_doc_group_mail_list" + ], + "desc": "Recipients when a perfmetrdir IETF Last Call review is completed", + "to": [ + "review_team_mail_list" + ] + }, + "model": "mailtrigger.mailtrigger", + "pk": "review_completed_perfmetrdir_lc" + }, + { + "fields": { + "cc": [ + "ietf_last_call", + "review_doc_all_parties", + "review_doc_group_mail_list" + ], + "desc": "Recipients when a perfmetrdir Telechat review is completed", + "to": [ + "review_team_mail_list" + ] + }, + "model": "mailtrigger.mailtrigger", + "pk": "review_completed_perfmetrdir_telechat" + }, { "fields": { "cc": [ @@ -6337,6 +7153,14 @@ "model": "mailtrigger.recipient", "pk": "liaison_coordinators" }, + { + "fields": { + "desc": "Email address of the formal sender of the statement", + "template": "{{liaison.from_contact}}" + }, + "model": "mailtrigger.recipient", + "pk": "liaison_from_contact" + }, { "fields": { "desc": "The assigned liaison manager for an external group ", @@ -6827,6 +7651,66 @@ "model": "name.appealartifacttypename", "pk": "response" }, + { + "fields": { + "desc": "", + "name": "ANRW Onsite", + "order": 0, + "used": true + }, + "model": "name.attendancetypename", + "pk": "anrw_onsite" + }, + { + "fields": { + "desc": "", + "name": "Hackathon Onsite", + "order": 0, + "used": true + }, + "model": "name.attendancetypename", + "pk": "hackathon_onsite" + }, + { + "fields": { + "desc": "", + "name": "Hackathon Remote", + "order": 0, + "used": true + }, + "model": "name.attendancetypename", + "pk": "hackathon_remote" + }, + { + "fields": { + "desc": "", + "name": "Onsite", + "order": 0, + "used": true + }, + "model": "name.attendancetypename", + "pk": "onsite" + }, + { + "fields": { + "desc": "", + "name": "Remote", + "order": 0, + "used": true + }, + "model": "name.attendancetypename", + "pk": "remote" + }, + { + "fields": { + "desc": "", + "name": "Unknown", + "order": 0, + "used": true + }, + "model": "name.attendancetypename", + "pk": "unknown" + }, { "fields": { "blocking": false, @@ -11270,6 +12154,17 @@ "model": "name.extresourcename", "pk": "mailing_list_archive" }, + { + "fields": { + "desc": "ORCID", + "name": "ORCID", + "order": 0, + "type": "url", + "used": true + }, + "model": "name.extresourcename", + "pk": "orcid" + }, { "fields": { "desc": "Related Implementations", @@ -12877,6 +13772,86 @@ "model": "name.proceedingsmaterialtypename", "pk": "wiki" }, + { + "fields": { + "desc": "", + "name": "ANRW Combo", + "order": 0, + "used": true + }, + "model": "name.registrationtickettypename", + "pk": "anrw_combo" + }, + { + "fields": { + "desc": "", + "name": "ANRW Only", + "order": 0, + "used": true + }, + "model": "name.registrationtickettypename", + "pk": "anrw_only" + }, + { + "fields": { + "desc": "", + "name": "Hackathon Combo", + "order": 0, + "used": true + }, + "model": "name.registrationtickettypename", + "pk": "hackathon_combo" + }, + { + "fields": { + "desc": "", + "name": "Hackathon Only", + "order": 0, + "used": true + }, + "model": "name.registrationtickettypename", + "pk": "hackathon_only" + }, + { + "fields": { + "desc": "", + "name": "One Day", + "order": 0, + "used": true + }, + "model": "name.registrationtickettypename", + "pk": "one_day" + }, + { + "fields": { + "desc": "", + "name": "Student", + "order": 0, + "used": true + }, + "model": "name.registrationtickettypename", + "pk": "student" + }, + { + "fields": { + "desc": "", + "name": "Unknown", + "order": 0, + "used": true + }, + "model": "name.registrationtickettypename", + "pk": "unknown" + }, + { + "fields": { + "desc": "", + "name": "Week Pass", + "order": 0, + "used": true + }, + "model": "name.registrationtickettypename", + "pk": "week_pass" + }, { "fields": { "desc": "The reviewer has accepted the assignment", @@ -13117,6 +14092,16 @@ "model": "name.reviewresultname", "pk": "almost-ready" }, + { + "fields": { + "desc": "", + "name": "Clarification Needed", + "order": 10, + "used": true + }, + "model": "name.reviewresultname", + "pk": "clarification-needed" + }, { "fields": { "desc": "", @@ -13422,7 +14407,7 @@ "desc": "", "name": "Liaison CC Contact", "order": 9, - "used": true + "used": false }, "model": "name.rolename", "pk": "liaison_cc_contact" @@ -13432,11 +14417,21 @@ "desc": "", "name": "Liaison Contact", "order": 8, - "used": true + "used": false }, "model": "name.rolename", "pk": "liaison_contact" }, + { + "fields": { + "desc": "Coordinates liaison handling for the IAB", + "name": "Liaison Coordinator", + "order": 14, + "used": true + }, + "model": "name.rolename", + "pk": "liaison_coordinator" + }, { "fields": { "desc": "", @@ -13522,7 +14517,7 @@ "desc": "Assigned permission TRAC_ADMIN in datatracker-managed Trac Wiki instances", "name": "Trac Admin", "order": 0, - "used": true + "used": false }, "model": "name.rolename", "pk": "trac-admin" @@ -13532,7 +14527,7 @@ "desc": "Provides log-in permission to restricted Trac instances. Used by the generate_apache_perms management command, called from ../../scripts/Cron-runner", "name": "Trac Editor", "order": 0, - "used": true + "used": false }, "model": "name.rolename", "pk": "trac-editor" @@ -13673,7 +14668,10 @@ "name": "Administrative", "on_agenda": true, "order": 5, - "timeslot_types": "[\n \"other\",\n \"reg\"\n]", + "timeslot_types": [ + "other", + "reg" + ], "used": true }, "model": "name.sessionpurposename", @@ -13685,7 +14683,10 @@ "name": "Closed meeting", "on_agenda": false, "order": 10, - "timeslot_types": "[\n \"other\",\n \"regular\"\n]", + "timeslot_types": [ + "other", + "regular" + ], "used": true }, "model": "name.sessionpurposename", @@ -13697,7 +14698,9 @@ "name": "Coding", "on_agenda": true, "order": 4, - "timeslot_types": "[\n \"other\"\n]", + "timeslot_types": [ + "other" + ], "used": true }, "model": "name.sessionpurposename", @@ -13709,7 +14712,7 @@ "name": "None", "on_agenda": true, "order": 0, - "timeslot_types": "[]", + "timeslot_types": [], "used": false }, "model": "name.sessionpurposename", @@ -13721,7 +14724,9 @@ "name": "Office hours", "on_agenda": true, "order": 3, - "timeslot_types": "[\n \"other\"\n]", + "timeslot_types": [ + "other" + ], "used": true }, "model": "name.sessionpurposename", @@ -13733,7 +14738,9 @@ "name": "Open meeting", "on_agenda": true, "order": 9, - "timeslot_types": "[\n \"other\"\n]", + "timeslot_types": [ + "other" + ], "used": true }, "model": "name.sessionpurposename", @@ -13745,7 +14752,9 @@ "name": "Plenary", "on_agenda": true, "order": 7, - "timeslot_types": "[\n \"plenary\"\n]", + "timeslot_types": [ + "plenary" + ], "used": true }, "model": "name.sessionpurposename", @@ -13757,7 +14766,10 @@ "name": "Presentation", "on_agenda": true, "order": 8, - "timeslot_types": "[\n \"other\",\n \"regular\"\n]", + "timeslot_types": [ + "other", + "regular" + ], "used": true }, "model": "name.sessionpurposename", @@ -13769,7 +14781,9 @@ "name": "Regular", "on_agenda": true, "order": 1, - "timeslot_types": "[\n \"regular\"\n]", + "timeslot_types": [ + "regular" + ], "used": true }, "model": "name.sessionpurposename", @@ -13781,7 +14795,10 @@ "name": "Social", "on_agenda": true, "order": 6, - "timeslot_types": "[\n \"break\",\n \"other\"\n]", + "timeslot_types": [ + "break", + "other" + ], "used": true }, "model": "name.sessionpurposename", @@ -13793,7 +14810,9 @@ "name": "Tutorial", "on_agenda": true, "order": 2, - "timeslot_types": "[\n \"other\"\n]", + "timeslot_types": [ + "other" + ], "used": true }, "model": "name.sessionpurposename", @@ -16802,5 +17821,13 @@ }, "model": "stats.countryalias", "pk": 303 + }, + { + "fields": { + "alias": "czechia", + "country": "CZ" + }, + "model": "stats.countryalias", + "pk": 304 } ] diff --git a/ietf/name/migrations/0016_attendancetypename_registrationtickettypename.py b/ietf/name/migrations/0016_attendancetypename_registrationtickettypename.py new file mode 100644 index 0000000000..9376d3a4c6 --- /dev/null +++ b/ietf/name/migrations/0016_attendancetypename_registrationtickettypename.py @@ -0,0 +1,47 @@ +# Generated by Django 4.2.17 on 2025-01-02 18:21 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("name", "0015_last_call_name"), + ] + + operations = [ + migrations.CreateModel( + name="AttendanceTypeName", + fields=[ + ( + "slug", + models.CharField(max_length=32, primary_key=True, serialize=False), + ), + ("name", models.CharField(max_length=255)), + ("desc", models.TextField(blank=True)), + ("used", models.BooleanField(default=True)), + ("order", models.IntegerField(default=0)), + ], + options={ + "ordering": ["order", "name"], + "abstract": False, + }, + ), + migrations.CreateModel( + name="RegistrationTicketTypeName", + fields=[ + ( + "slug", + models.CharField(max_length=32, primary_key=True, serialize=False), + ), + ("name", models.CharField(max_length=255)), + ("desc", models.TextField(blank=True)), + ("used", models.BooleanField(default=True)), + ("order", models.IntegerField(default=0)), + ], + options={ + "ordering": ["order", "name"], + "abstract": False, + }, + ), + ] diff --git a/ietf/name/migrations/0017_populate_new_reg_names.py b/ietf/name/migrations/0017_populate_new_reg_names.py new file mode 100644 index 0000000000..51954885c0 --- /dev/null +++ b/ietf/name/migrations/0017_populate_new_reg_names.py @@ -0,0 +1,39 @@ +# Generated by Django 4.2.17 on 2025-01-02 18:26 + +from django.db import migrations + +def forward(apps, schema_editor): + AttendanceTypeName = apps.get_model('name', 'AttendanceTypeName') + RegistrationTicketTypeName = apps.get_model('name', 'RegistrationTicketTypeName') + AttendanceTypeName.objects.create(slug='onsite', name='Onsite') + AttendanceTypeName.objects.create(slug='remote', name='Remote') + AttendanceTypeName.objects.create(slug='hackathon_onsite', name='Hackathon Onsite') + AttendanceTypeName.objects.create(slug='hackathon_remote', name='Hackathon Remote') + AttendanceTypeName.objects.create(slug='anrw_onsite', name='ANRW Onsite') + AttendanceTypeName.objects.create(slug='unknown', name='Unknown') + RegistrationTicketTypeName.objects.create(slug='week_pass', name='Week Pass') + RegistrationTicketTypeName.objects.create(slug='one_day', name='One Day') + RegistrationTicketTypeName.objects.create(slug='student', name='Student') + RegistrationTicketTypeName.objects.create(slug='hackathon_only', name='Hackathon Only') + RegistrationTicketTypeName.objects.create(slug='hackathon_combo', name='Hackathon Combo') + RegistrationTicketTypeName.objects.create(slug='anrw_only', name='ANRW Only') + RegistrationTicketTypeName.objects.create(slug='anrw_combo', name='ANRW Combo') + RegistrationTicketTypeName.objects.create(slug='unknown', name='Unknown') + + +def reverse(apps, schema_editor): + AttendanceTypeName = apps.get_model('name', 'AttendanceTypeName') + RegistrationTicketTypeName = apps.get_model('name', 'RegistrationTicketTypeName') + AttendanceTypeName.objects.delete() + RegistrationTicketTypeName.objects.delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ("name", "0016_attendancetypename_registrationtickettypename"), + ] + + operations = [ + migrations.RunPython(forward, reverse), + ] diff --git a/ietf/name/migrations/0018_alter_rolenames.py b/ietf/name/migrations/0018_alter_rolenames.py new file mode 100644 index 0000000000..f931de2e97 --- /dev/null +++ b/ietf/name/migrations/0018_alter_rolenames.py @@ -0,0 +1,36 @@ +# Copyright The IETF Trust 2025, All Rights Reserved# Generated by Django 4.2.21 on 2025-05-30 16:35 + +from django.db import migrations + + +def forward(apps, schema_editor): + RoleName = apps.get_model("name", "RoleName") + RoleName.objects.filter(slug__in=["liaison_contact", "liaison_cc_contact"]).update( + used=False + ) + RoleName.objects.get_or_create( + slug="liaison_coordinator", + defaults={ + "name": "Liaison Coordinator", + "desc": "Coordinates liaison handling for the IAB", + "order": 14, + }, + ) + RoleName.objects.filter(slug__contains="trac-").update(used=False) + + +def reverse(apps, schema_editor): + RoleName = apps.get_model("name", "RoleName") + RoleName.objects.filter(slug__in=["liaison_contact", "liaison_cc_contact"]).update( + used=True + ) + RoleName.objects.filter(slug="liaison_coordinator").delete() + # Intentionally not restoring trac-* RoleNames to used=True + + +class Migration(migrations.Migration): + dependencies = [ + ("name", "0017_populate_new_reg_names"), + ] + + operations = [migrations.RunPython(forward, reverse)] diff --git a/ietf/name/migrations/0019_alter_sessionpurposename_timeslot_types.py b/ietf/name/migrations/0019_alter_sessionpurposename_timeslot_types.py new file mode 100644 index 0000000000..a0ca81836d --- /dev/null +++ b/ietf/name/migrations/0019_alter_sessionpurposename_timeslot_types.py @@ -0,0 +1,27 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations, models +import ietf.utils.validators + + +class Migration(migrations.Migration): + dependencies = [ + ("name", "0018_alter_rolenames"), + ] + + operations = [ + migrations.AlterField( + model_name="sessionpurposename", + name="timeslot_types", + field=models.JSONField( + default=list, + help_text="Allowed TimeSlotTypeNames", + max_length=256, + validators=[ + ietf.utils.validators.JSONForeignKeyListValidator( + "name.TimeSlotTypeName" + ) + ], + ), + ), + ] diff --git a/ietf/name/models.py b/ietf/name/models.py index 8c2520a489..24104c5f45 100644 --- a/ietf/name/models.py +++ b/ietf/name/models.py @@ -1,8 +1,6 @@ # Copyright The IETF Trust 2010-2020, All Rights Reserved # -*- coding: utf-8 -*- -import jsonfield - from django.db import models from ietf.utils.models import ForeignKey @@ -73,8 +71,8 @@ class SessionStatusName(NameModel): """Waiting for Approval, Approved, Waiting for Scheduling, Scheduled, Cancelled, Disapproved""" class SessionPurposeName(NameModel): """Regular, Tutorial, Office Hours, Coding, Social, Admin""" - timeslot_types = jsonfield.JSONField( - max_length=256, blank=False, default=[], + timeslot_types = models.JSONField( + max_length=256, blank=False, default=list, help_text='Allowed TimeSlotTypeNames', validators=[JSONForeignKeyListValidator('name.TimeSlotTypeName')], ) @@ -151,6 +149,9 @@ class SlideSubmissionStatusName(NameModel): "Pending, Accepted, Rejected" class TelechatAgendaSectionName(NameModel): """roll_call, minutes, action_items""" - class AppealArtifactTypeName(NameModel): pass +class AttendanceTypeName(NameModel): + """onsite, remote, hackathon_onsite, hackathon_remote""" +class RegistrationTicketTypeName(NameModel): + """week, one_day, student""" diff --git a/ietf/name/resources.py b/ietf/name/resources.py index dffa7669db..0cb0e41e0b 100644 --- a/ietf/name/resources.py +++ b/ietf/name/resources.py @@ -19,7 +19,7 @@ RoleName, RoomResourceName, SessionStatusName, StdLevelName, StreamName, TimeSlotTypeName, TopicAudienceName, ReviewerQueuePolicyName, TimerangeName, ExtResourceTypeName, ExtResourceName, SlideSubmissionStatusName, ProceedingsMaterialTypeName, SessionPurposeName, TelechatAgendaSectionName, - AppealArtifactTypeName ) + AppealArtifactTypeName, AttendanceTypeName, RegistrationTicketTypeName ) class TimeSlotTypeNameResource(ModelResource): class Meta: @@ -752,3 +752,33 @@ class Meta: "order": ALL, } api.name.register(AppealArtifactTypeNameResource()) + + +class AttendanceTypeNameResource(ModelResource): + class Meta: + cache = SimpleCache() + queryset = AttendanceTypeName.objects.all() + serializer = api.Serializer() + filtering = { + "slug": ALL, + "name": ALL, + "desc": ALL, + "used": ALL, + "order": ALL, + } +api.name.register(AttendanceTypeNameResource()) + + +class RegistrationTicketTypeNameResource(ModelResource): + class Meta: + cache = SimpleCache() + queryset = RegistrationTicketTypeName.objects.all() + serializer = api.Serializer() + filtering = { + "slug": ALL, + "name": ALL, + "desc": ALL, + "used": ALL, + "order": ALL, + } +api.name.register(RegistrationTicketTypeNameResource()) diff --git a/ietf/name/serializers.py b/ietf/name/serializers.py new file mode 100644 index 0000000000..a764f56051 --- /dev/null +++ b/ietf/name/serializers.py @@ -0,0 +1,11 @@ +# Copyright The IETF Trust 2024, All Rights Reserved +"""django-rest-framework serializers""" +from rest_framework import serializers + +from .models import StreamName + + +class StreamNameSerializer(serializers.ModelSerializer): + class Meta: + model = StreamName + fields = ["slug", "name", "desc"] diff --git a/ietf/nomcom/management/tests.py b/ietf/nomcom/management/tests.py index 7bda2b5aa5..08c0e1fe32 100644 --- a/ietf/nomcom/management/tests.py +++ b/ietf/nomcom/management/tests.py @@ -1,7 +1,7 @@ # Copyright The IETF Trust 2021, All Rights Reserved # -*- coding: utf-8 -*- """Tests of nomcom management commands""" -import mock +from unittest import mock import sys from collections import namedtuple diff --git a/ietf/nomcom/tests.py b/ietf/nomcom/tests.py index 8f94cc7fc5..210788ce07 100644 --- a/ietf/nomcom/tests.py +++ b/ietf/nomcom/tests.py @@ -1,10 +1,9 @@ -# Copyright The IETF Trust 2012-2023, All Rights Reserved -# -*- coding: utf-8 -*- +# Copyright The IETF Trust 2012-2025, All Rights Reserved import datetime import io -import mock +from unittest import mock import random import shutil @@ -27,11 +26,18 @@ from ietf.api.views import EmailIngestionError from ietf.dbtemplate.factories import DBTemplateFactory from ietf.dbtemplate.models import DBTemplate -from ietf.doc.factories import DocEventFactory, WgDocumentAuthorFactory, \ - NewRevisionDocEventFactory, DocumentAuthorFactory +from ietf.doc.factories import ( + DocEventFactory, + WgDocumentAuthorFactory, + NewRevisionDocEventFactory, + DocumentAuthorFactory, + RfcAuthorFactory, + WgDraftFactory, WgRfcFactory, +) from ietf.group.factories import GroupFactory, GroupHistoryFactory, RoleFactory, RoleHistoryFactory from ietf.group.models import Group, Role -from ietf.meeting.factories import MeetingFactory, AttendedFactory +from ietf.meeting.factories import MeetingFactory, AttendedFactory, RegistrationFactory +from ietf.meeting.models import Registration from ietf.message.models import Message from ietf.nomcom.test_data import nomcom_test_data, generate_cert, check_comments, \ COMMUNITY_USER, CHAIR_USER, \ @@ -44,14 +50,22 @@ nomcom_kwargs_for_year, provide_private_key_to_test_client, \ key from ietf.nomcom.tasks import send_nomcom_reminders_task -from ietf.nomcom.utils import get_nomcom_by_year, make_nomineeposition, \ - get_hash_nominee_position, is_eligible, list_eligible, \ - get_eligibility_date, suggest_affiliation, ingest_feedback_email, \ - decorate_volunteers_with_qualifications, send_reminders, _is_time_to_send_reminder +from ietf.nomcom.utils import ( + get_nomcom_by_year, + make_nomineeposition, + get_hash_nominee_position, + is_eligible, + list_eligible, + get_eligibility_date, + suggest_affiliation, + ingest_feedback_email, + decorate_volunteers_with_qualifications, + send_reminders, + _is_time_to_send_reminder, + get_qualified_author_queryset, +) from ietf.person.factories import PersonFactory, EmailFactory from ietf.person.models import Email, Person -from ietf.stats.models import MeetingRegistration -from ietf.stats.factories import MeetingRegistrationFactory from ietf.utils.mail import outbox, empty_outbox, get_payload_text from ietf.utils.test_utils import login_testing_unauthorized, TestCase, unicontent from ietf.utils.timezone import date_today, datetime_today, datetime_from_date, DEADLINE_TZINFO @@ -2061,7 +2075,15 @@ def first_meeting_of_year(year): if not ' ' in ascii: continue first_name, last_name = ascii.rsplit(None, 1) - MeetingRegistration.objects.create(meeting=meeting, first_name=first_name, last_name=last_name, person=person, country_code='WO', email=email, attended=True) + RegistrationFactory( + meeting=meeting, + first_name=first_name, + last_name=last_name, + person=person, + country_code='WO', + email=email, + attended=True + ) for view in ('public_eligible','private_eligible'): url = reverse(f'ietf.nomcom.views.{view}',kwargs={'year':self.nc.year()}) for username in (self.chair.user.username,'secretary'): @@ -2084,7 +2106,7 @@ def first_meeting_of_year(year): for number in range(meeting_start, meeting_start+8): m = MeetingFactory.create(type_id='ietf', number=number) for p in people: - m.meetingregistration_set.create(person=p, reg_type="onsite", checkedin=True, attended=True) + RegistrationFactory(meeting=m, person=p, checkedin=True, attended=True) for p in people: self.nc.volunteer_set.create(person=p,affiliation='something') for view in ('public_volunteers','private_volunteers'): @@ -2110,10 +2132,6 @@ def first_meeting_of_year(year): self.assertContains(response, people[-1].plain_name(), status_code=200) self.assertNotContains(response, unqualified_person.plain_name()) - - - - class NomComIndexTests(TestCase): def setUp(self): super().setUp() @@ -2437,6 +2455,85 @@ def test_get_eligibility_date(self): NomComFactory(group__acronym=f'nomcom{this_year}', first_call_for_volunteers=datetime.date(this_year,5,6)) self.assertEqual(get_eligibility_date(),datetime.date(this_year,5,6)) + def test_get_qualified_author_queryset(self): + """get_qualified_author_queryset implements the eligiblity rules correctly + + This is not an exhaustive test of corner cases. Overlaps considerably with + rfc8989EligibilityTests.test_elig_by_author(). + """ + people = PersonFactory.create_batch(2) + extra_person = PersonFactory() + base_qs = Person.objects.filter(pk__in=[person.pk for person in people]) + now = datetime.datetime.now(tz=datetime.UTC) + one_year = datetime.timedelta(days=365) + + # Authors with no qualifying drafts + self.assertCountEqual( + get_qualified_author_queryset(base_qs, now - 5 * one_year, now), [] + ) + + # Authors with one qualifying draft + approved_draft = WgDraftFactory(authors=people, states=[("draft", "active")]) + DocEventFactory( + type="iesg_approved", + doc=approved_draft, + time=now - 4 * one_year, + ) + self.assertCountEqual( + get_qualified_author_queryset(base_qs, now - 5 * one_year, now), [] + ) + + # Create a draft that was published into an RFC. Give it an extra author who + # should not be eligible. + published_draft = WgDraftFactory(authors=people, states=[("draft", "rfc")]) + DocEventFactory( + type="iesg_approved", + doc=published_draft, + time=now - 5.5 * one_year, # < 6 years ago + ) + rfc = WgRfcFactory( + authors=people + [extra_person], + group=published_draft.group, + ) + DocEventFactory( + type="published_rfc", + doc=rfc, + time=now - 0.5 * one_year, # < 1 year ago + ) + # Period 6 years ago to 1 year ago - authors are eligible due to the + # iesg-approved draft in this window + self.assertCountEqual( + get_qualified_author_queryset(base_qs, now - 6 * one_year, now - one_year), + people, + ) + + # Period 5 years ago to now - authors are eligible due to the RFC publication + self.assertCountEqual( + get_qualified_author_queryset(base_qs, now - 5 * one_year, now), + people, + ) + + # Use the extra_person to check that a single doc can't count both as an + # RFC _and_ an approved draft. Use an eligibility interval that includes both + # the approval and the RFC publication + self.assertCountEqual( + get_qualified_author_queryset(base_qs, now - 6 * one_year, now), + people, # does not include extra_person! + ) + + # Now add an RfcAuthor for only one of the two authors to the RFC. This should + # remove the other author from the eligibility list because the DocumentAuthor + # records are no longer used. + RfcAuthorFactory( + document=rfc, + person=people[0], + titlepage_name="P. Zero", + ) + self.assertCountEqual( + get_qualified_author_queryset(base_qs, now - 5 * one_year, now), + [people[0]], + ) + class rfc8713EligibilityTests(TestCase): @@ -2460,7 +2557,7 @@ def setUp(self): for combo in combinations(meetings,combo_len): p = PersonFactory() for m in combo: - MeetingRegistrationFactory(person=p, meeting=m, attended=True) + RegistrationFactory(person=p, meeting=m, attended=True) if combo_len<3: self.ineligible_people.append(p) else: @@ -2470,7 +2567,7 @@ def setUp(self): def ineligible_person_with_role(**kwargs): p = RoleFactory(**kwargs).person for m in meetings: - MeetingRegistrationFactory(person=p, meeting=m, attended=True) + RegistrationFactory(person=p, meeting=m, attended=True) self.ineligible_people.append(p) for group in ['isocbot', 'ietf-trust', 'llc-board', 'iab']: for role in ['member', 'chair']: @@ -2485,8 +2582,7 @@ def ineligible_person_with_role(**kwargs): self.other_date = datetime.date(2009,5,1) self.other_people = PersonFactory.create_batch(1) for date in (datetime.date(2009,3,1), datetime.date(2008,11,1), datetime.date(2008,7,1)): - MeetingRegistrationFactory(person=self.other_people[0],meeting__date=date, meeting__type_id='ietf', attended=True) - + RegistrationFactory(person=self.other_people[0], meeting__date=date, meeting__type_id='ietf', attended=True) def test_is_person_eligible(self): for person in self.eligible_people: @@ -2530,7 +2626,7 @@ def setUp(self): for combo in combinations(meetings,combo_len): p = PersonFactory() for m in combo: - MeetingRegistrationFactory(person=p, meeting=m, attended=True) + RegistrationFactory(person=p, meeting=m, attended=True) if combo_len<3: self.ineligible_people.append(p) else: @@ -2578,7 +2674,7 @@ def test_elig_by_meetings(self): for combo in combinations(prev_five,combo_len): p = PersonFactory() for m in combo: - MeetingRegistrationFactory(person=p, meeting=m, attended=True) # not checkedin because this forces looking at older meetings + RegistrationFactory(person=p, meeting=m, attended=True) # not checkedin because this forces looking at older meetings AttendedFactory(session__meeting=m, session__type_id='plenary',person=p) if combo_len<3: ineligible_people.append(p) @@ -2593,8 +2689,9 @@ def test_elig_by_meetings(self): for person in ineligible_people: self.assertFalse(is_eligible(person,nomcom)) - Person.objects.filter(pk__in=[p.pk for p in eligible_people+ineligible_people]).delete() - + people = Person.objects.filter(pk__in=[p.pk for p in eligible_people + ineligible_people]) + Registration.objects.filter(person__in=people).delete() + people.delete() def test_elig_by_office_active_groups(self): @@ -2721,33 +2818,41 @@ def test_elig_by_author(self): ineligible = set() p = PersonFactory() - ineligible.add(p) - + ineligible.add(p) # no RFCs or iesg-approved drafts p = PersonFactory() - da = WgDocumentAuthorFactory(person=p) - DocEventFactory(type='published_rfc',doc=da.document,time=middle_date) - ineligible.add(p) + doc = WgRfcFactory(authors=[p]) + DocEventFactory(type='published_rfc', doc=doc, time=middle_date) + ineligible.add(p) # only one RFC p = PersonFactory() - da = WgDocumentAuthorFactory(person=p) + da = WgDocumentAuthorFactory( + person=p, + document__states=[("draft", "active"), ("draft-rfceditor", "ref")], + ) DocEventFactory(type='iesg_approved',doc=da.document,time=last_date) - da = WgDocumentAuthorFactory(person=p) - DocEventFactory(type='published_rfc',doc=da.document,time=first_date) - eligible.add(p) + doc = WgRfcFactory(authors=[p]) + DocEventFactory(type='published_rfc', doc=doc, time=first_date) + eligible.add(p) # one RFC and one iesg-approved draft p = PersonFactory() - da = WgDocumentAuthorFactory(person=p) + da = WgDocumentAuthorFactory( + person=p, + document__states=[("draft", "active"), ("draft-rfceditor", "ref")], + ) DocEventFactory(type='iesg_approved',doc=da.document,time=middle_date) - da = WgDocumentAuthorFactory(person=p) - DocEventFactory(type='published_rfc',doc=da.document,time=day_before_first_date) - ineligible.add(p) + doc = WgRfcFactory(authors=[p]) + DocEventFactory(type='published_rfc', doc=doc, time=day_before_first_date) + ineligible.add(p) # RFC is out of the eligibility window p = PersonFactory() - da = WgDocumentAuthorFactory(person=p) + da = WgDocumentAuthorFactory( + person=p, + document__states=[("draft", "active"), ("draft-rfceditor", "ref")], + ) DocEventFactory(type='iesg_approved',doc=da.document,time=day_after_last_date) - da = WgDocumentAuthorFactory(person=p) - DocEventFactory(type='published_rfc',doc=da.document,time=middle_date) - ineligible.add(p) + doc = WgRfcFactory(authors=[p]) + DocEventFactory(type='published_rfc', doc=doc, time=middle_date) + ineligible.add(p) # iesg approval is outside the eligibility window for person in eligible: self.assertTrue(is_eligible(person,nomcom)) @@ -2778,7 +2883,7 @@ def setUp(self): def test_registration_is_not_enough(self): p = PersonFactory() for meeting in self.meetings: - MeetingRegistrationFactory(person=p, meeting=meeting, checkedin=False) + RegistrationFactory(person=p, meeting=meeting, checkedin=False) self.assertFalse(is_eligible(p, self.nomcom)) def test_elig_by_meetings(self): @@ -2795,7 +2900,7 @@ def test_elig_by_meetings(self): for method in attendance_methods: p = PersonFactory() for meeting in combo: - MeetingRegistrationFactory(person=p, meeting=meeting, reg_type='onsite', checkedin=(method in ('checkedin', 'both'))) + RegistrationFactory(person=p, meeting=meeting, checkedin=(method in ('checkedin', 'both'))) if method in ('session', 'both'): AttendedFactory(session__meeting=meeting, session__type_id='plenary',person=p) if combo_len<3: @@ -2828,7 +2933,7 @@ def test_volunteer(self): self.assertContains(r, 'NomCom is not accepting volunteers at this time', status_code=200) nomcom.is_accepting_volunteers = True nomcom.save() - MeetingRegistrationFactory(person=person, affiliation='mtg_affiliation', checkedin=True) + RegistrationFactory(person=person, affiliation='mtg_affiliation', checkedin=True) r = self.client.get(url) self.assertContains(r, 'Volunteer for NomCom', status_code=200) self.assertContains(r, 'mtg_affiliation') @@ -2875,15 +2980,38 @@ def test_volunteer(self): def test_suggest_affiliation(self): person = PersonFactory() - self.assertEqual(suggest_affiliation(person), '') - da = DocumentAuthorFactory(person=person,affiliation='auth_affil') + self.assertEqual(suggest_affiliation(person), "") + rfc_da = DocumentAuthorFactory( + person=person, + document__type_id="rfc", + affiliation="", + ) + rfc = rfc_da.document + DocEventFactory(doc=rfc, type="published_rfc") + self.assertEqual(suggest_affiliation(person), "") + + rfc_da.affiliation = "rfc_da_affil" + rfc_da.save() + self.assertEqual(suggest_affiliation(person), "rfc_da_affil") + + rfc_ra = RfcAuthorFactory(person=person, document=rfc, affiliation="") + self.assertEqual(suggest_affiliation(person), "") + + rfc_ra.affiliation = "rfc_ra_affil" + rfc_ra.save() + self.assertEqual(suggest_affiliation(person), "rfc_ra_affil") + + da = DocumentAuthorFactory(person=person, affiliation="auth_affil") NewRevisionDocEventFactory(doc=da.document) - self.assertEqual(suggest_affiliation(person), 'auth_affil') + self.assertEqual(suggest_affiliation(person), "auth_affil") + nc = NomComFactory() - nc.volunteer_set.create(person=person,affiliation='volunteer_affil') - self.assertEqual(suggest_affiliation(person), 'volunteer_affil') - MeetingRegistrationFactory(person=person, affiliation='meeting_affil') - self.assertEqual(suggest_affiliation(person), 'meeting_affil') + nc.volunteer_set.create(person=person, affiliation="volunteer_affil") + self.assertEqual(suggest_affiliation(person), "volunteer_affil") + + RegistrationFactory(person=person, affiliation="meeting_affil") + self.assertEqual(suggest_affiliation(person), "meeting_affil") + class VolunteerDecoratorUnitTests(TestCase): def test_decorate_volunteers_with_qualifications(self): @@ -2900,7 +3028,7 @@ def test_decorate_volunteers_with_qualifications(self): ('106', datetime.date(2019, 11, 16)), ]] for m in meetings: - MeetingRegistrationFactory(meeting=m, person=meeting_person, attended=True) + RegistrationFactory(meeting=m, person=meeting_person, attended=True) AttendedFactory(session__meeting=m, session__type_id='plenary', person=meeting_person) nomcom.volunteer_set.create(person=meeting_person) @@ -2919,15 +3047,15 @@ def test_decorate_volunteers_with_qualifications(self): author_person = PersonFactory() for i in range(2): - da = WgDocumentAuthorFactory(person=author_person) + doc = WgRfcFactory(authors=[author_person]) DocEventFactory( type='published_rfc', - doc=da.document, + doc=doc, time=datetime.datetime( elig_date.year - 3, elig_date.month, 28 if elig_date.month == 2 and elig_date.day == 29 else elig_date.day, - tzinfo=datetime.timezone.utc, + tzinfo=datetime.UTC, ) ) nomcom.volunteer_set.create(person=author_person) diff --git a/ietf/nomcom/utils.py b/ietf/nomcom/utils.py index ab155ef1d5..a2ab680df6 100644 --- a/ietf/nomcom/utils.py +++ b/ietf/nomcom/utils.py @@ -18,7 +18,7 @@ from email.utils import parseaddr from textwrap import dedent -from django.db.models import Q, Count +from django.db.models import Q, Count, F, QuerySet from django.conf import settings from django.contrib.sites.models import Site from django.core.exceptions import ObjectDoesNotExist @@ -27,7 +27,7 @@ from django.shortcuts import get_object_or_404 from ietf.dbtemplate.models import DBTemplate -from ietf.doc.models import DocEvent, NewRevisionDocEvent +from ietf.doc.models import DocEvent, NewRevisionDocEvent, Document from ietf.group.models import Group, Role from ietf.person.models import Email, Person from ietf.mailtrigger.utils import gather_address_lists @@ -184,7 +184,7 @@ def retrieve_nomcom_private_key(request, year): if not private_key: return private_key - command = "%s bf -d -in /dev/stdin -k \"%s\" -a" + command = "%s aes-128-ecb -d -in /dev/stdin -k \"%s\" -a -iter 1000" code, out, error = pipe( command % ( settings.OPENSSL_COMMAND, @@ -208,7 +208,7 @@ def store_nomcom_private_key(request, year, private_key): if not private_key: request.session['NOMCOM_PRIVATE_KEY_%s' % year] = '' else: - command = "%s bf -e -in /dev/stdin -k \"%s\" -a" + command = "%s aes-128-ecb -e -in /dev/stdin -k \"%s\" -a -iter 1000" code, out, error = pipe( command % ( settings.OPENSSL_COMMAND, @@ -576,6 +576,70 @@ def get_8989_eligibility_querysets(date, base_qs): def get_9389_eligibility_querysets(date, base_qs): return get_threerule_eligibility_querysets(date, base_qs, three_of_five_callable=three_of_five_eligible_9389) + +def get_qualified_author_queryset( + base_qs: QuerySet[Person], + eligibility_period_start: datetime.datetime, + eligibility_period_end: datetime.datetime, +): + """Filter a Person queryset, keeping those qualified by RFC 8989's author path + + The author path is defined by "path 3" in section 4 of RFC 8989. It qualifies + a person who has been a front-page listed author or editor of at least two IETF- + stream RFCs within the last five years. An I-D in the RFC Editor queue that was + approved by the IESG is treated as an RFC, using the date of entry to the RFC + Editor queue as the date for qualification. + + This method does not strictly enforce "in the RFC Editor queue" for IESG-approved + drafts when computing eligibility. In the overwhelming majority of cases, an IESG- + approved draft immediately enters the queue and goes on to be published, so this + simplification makes the calculation much easier and virtually never affects + eligibility. + + Arguments eligibility_period_start and eligibility_period_end are datetimes that + mark the start and end of the eligibility period. These should be five years apart. + """ + # First, get the RFCs using publication date + qualifying_rfc_pub_events = DocEvent.objects.filter( + type='published_rfc', + time__gte=eligibility_period_start, + time__lte=eligibility_period_end, + ) + qualifying_rfcs = Document.objects.filter( + type_id="rfc", + docevent__in=qualifying_rfc_pub_events + ).annotate( + rfcauthor_count=Count("rfcauthor") + ) + rfcs_with_rfcauthors = qualifying_rfcs.filter(rfcauthor_count__gt=0).distinct() + rfcs_without_rfcauthors = qualifying_rfcs.filter(rfcauthor_count=0).distinct() + + # Second, get the IESG-approved I-Ds excluding any we're already counting as rfcs + qualifying_approval_events = DocEvent.objects.filter( + type='iesg_approved', + time__gte=eligibility_period_start, + time__lte=eligibility_period_end, + ) + qualifying_drafts = Document.objects.filter( + type_id="draft", + docevent__in=qualifying_approval_events, + ).exclude( + relateddocument__relationship_id="became_rfc", + relateddocument__target__in=qualifying_rfcs, + ).distinct() + + return base_qs.filter( + Q(documentauthor__document__in=qualifying_drafts) + | Q(rfcauthor__document__in=rfcs_with_rfcauthors) + | Q(documentauthor__document__in=rfcs_without_rfcauthors) + ).annotate( + document_author_count=Count('documentauthor'), + rfc_author_count=Count("rfcauthor") + ).annotate( + authorship_count=F("document_author_count") + F("rfc_author_count") + ).filter(authorship_count__gte=2) + + def get_threerule_eligibility_querysets(date, base_qs, three_of_five_callable): if not base_qs: base_qs = Person.objects.all() @@ -608,14 +672,7 @@ def get_threerule_eligibility_querysets(date, base_qs, three_of_five_callable): ) ).distinct() - rfc_pks = set(DocEvent.objects.filter(type='published_rfc', time__gte=five_years_ago, time__lte=date_as_dt).values_list('doc__pk', flat=True)) - iesgappr_pks = set(DocEvent.objects.filter(type='iesg_approved', time__gte=five_years_ago, time__lte=date_as_dt).values_list('doc__pk',flat=True)) - qualifying_pks = rfc_pks.union(iesgappr_pks.difference(rfc_pks)) - author_qs = base_qs.filter( - documentauthor__document__pk__in=qualifying_pks - ).annotate( - document_author_count = Count('documentauthor') - ).filter(document_author_count__gte=2) + author_qs = get_qualified_author_queryset(base_qs, five_years_ago, date_as_dt) return three_of_five_qs, officer_qs, author_qs def list_eligible_8989(date, base_qs=None): @@ -666,14 +723,14 @@ def previous_five_meetings(date = None): return Meeting.objects.filter(type='ietf',date__lte=date).order_by('-date')[:5] def three_of_five_eligible_8713(previous_five, queryset=None): - """ Return a list of Person records who attended at least + """ Return a list of Person records who attended at least 3 of the 5 type_id='ietf' meetings before the given date. Does not disqualify anyone based on held roles. This variant bases the calculation on MeetingRegistration.attended """ if queryset is None: queryset = Person.objects.all() - return queryset.filter(meetingregistration__meeting__in=list(previous_five),meetingregistration__attended=True).annotate(mtg_count=Count('meetingregistration')).filter(mtg_count__gte=3) + return queryset.filter(registration__meeting__in=list(previous_five), registration__attended=True).annotate(mtg_count=Count('registration')).filter(mtg_count__gte=3) def three_of_five_eligible_9389(previous_five, queryset=None): """ Return a list of Person records who attended at least @@ -691,18 +748,42 @@ def three_of_five_eligible_9389(previous_five, queryset=None): counts[id] += 1 return queryset.filter(pk__in=[id for id, count in counts.items() if count >= 3]) -def suggest_affiliation(person): - recent_meeting = person.meetingregistration_set.order_by('-meeting__date').first() - affiliation = recent_meeting.affiliation if recent_meeting else '' - if not affiliation: - recent_volunteer = person.volunteer_set.order_by('-nomcom__group__acronym').first() - if recent_volunteer: - affiliation = recent_volunteer.affiliation - if not affiliation: - recent_draft_revision = NewRevisionDocEvent.objects.filter(doc__type_id='draft',doc__documentauthor__person=person).order_by('-time').first() - if recent_draft_revision: - affiliation = recent_draft_revision.doc.documentauthor_set.filter(person=person).first().affiliation - return affiliation +def suggest_affiliation(person) -> str: + """Heuristically suggest a current affiliation for a Person""" + recent_meeting = person.registration_set.order_by('-meeting__date').first() + if recent_meeting and recent_meeting.affiliation: + return recent_meeting.affiliation + + recent_volunteer = person.volunteer_set.order_by('-nomcom__group__acronym').first() + if recent_volunteer and recent_volunteer.affiliation: + return recent_volunteer.affiliation + + recent_draft_revision = NewRevisionDocEvent.objects.filter( + doc__type_id="draft", + doc__documentauthor__person=person, + ).order_by("-time").first() + if recent_draft_revision: + draft_author = recent_draft_revision.doc.documentauthor_set.filter( + person=person + ).first() + if draft_author and draft_author.affiliation: + return draft_author.affiliation + + recent_rfc_publication = DocEvent.objects.filter( + Q(doc__documentauthor__person=person) | Q(doc__rfcauthor__person=person), + doc__type_id="rfc", + type="published_rfc", + ).order_by("-time").first() + if recent_rfc_publication: + rfc = recent_rfc_publication.doc + if rfc.rfcauthor_set.exists(): + rfc_author = rfc.rfcauthor_set.filter(person=person).first() + else: + rfc_author = rfc.documentauthor_set.filter(person=person).first() + if rfc_author and rfc_author.affiliation: + return rfc_author.affiliation + return "" + def extract_volunteers(year): nomcom = get_nomcom_by_year(year) diff --git a/ietf/nomcom/views.py b/ietf/nomcom/views.py index d34126b1e7..3f90be5253 100644 --- a/ietf/nomcom/views.py +++ b/ietf/nomcom/views.py @@ -981,7 +981,7 @@ def view_feedback_topic(request, year, topic_id): reviewer = request.user.person last_seen = TopicFeedbackLastSeen.objects.filter(reviewer=reviewer,topic=topic).first() - last_seen_time = (last_seen and last_seen.time) or datetime.datetime(year=1, month=1, day=1, tzinfo=datetime.timezone.utc) + last_seen_time = (last_seen and last_seen.time) or datetime.datetime(year=1, month=1, day=1, tzinfo=datetime.UTC) if last_seen: last_seen.save() else: @@ -1017,7 +1017,10 @@ def view_feedback_nominee(request, year, nominee_id): 'positions': ','.join([str(p) for p in feedback.positions.all()]), }, request=request) - response = HttpResponse(response, content_type='text/plain') + response = HttpResponse( + response, + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + ) response['Content-Disposition'] = f'attachment; filename="{fn}"' return response elif submit == 'reclassify': @@ -1041,7 +1044,7 @@ def view_feedback_nominee(request, year, nominee_id): }) last_seen = FeedbackLastSeen.objects.filter(reviewer=reviewer,nominee=nominee).first() - last_seen_time = (last_seen and last_seen.time) or datetime.datetime(year=1, month=1, day=1, tzinfo=datetime.timezone.utc) + last_seen_time = (last_seen and last_seen.time) or datetime.datetime(year=1, month=1, day=1, tzinfo=datetime.UTC) if last_seen: last_seen.save() else: diff --git a/ietf/person/admin.py b/ietf/person/admin.py index cd8ca2abf1..f46edcf8ae 100644 --- a/ietf/person/admin.py +++ b/ietf/person/admin.py @@ -7,6 +7,7 @@ from ietf.person.models import Email, Alias, Person, PersonalApiKey, PersonEvent, PersonApiKeyEvent, PersonExtResource from ietf.person.name import name_parts +from ietf.utils.admin import SaferStackedInline, SaferTabularInline from ietf.utils.validators import validate_external_resource_value @@ -16,7 +17,7 @@ class EmailAdmin(simple_history.admin.SimpleHistoryAdmin): search_fields = ["address", "person__name", ] admin.site.register(Email, EmailAdmin) -class EmailInline(admin.TabularInline): +class EmailInline(SaferTabularInline): model = Email class AliasAdmin(admin.ModelAdmin): @@ -25,7 +26,7 @@ class AliasAdmin(admin.ModelAdmin): raw_id_fields = ["person"] admin.site.register(Alias, AliasAdmin) -class AliasInline(admin.StackedInline): +class AliasInline(SaferStackedInline): model = Alias class PersonAdmin(simple_history.admin.SimpleHistoryAdmin): diff --git a/ietf/person/forms.py b/ietf/person/forms.py index 81ee362561..7eef8aa17b 100644 --- a/ietf/person/forms.py +++ b/ietf/person/forms.py @@ -1,15 +1,26 @@ -# Copyright The IETF Trust 2018-2020, All Rights Reserved +# Copyright The IETF Trust 2018-2025, All Rights Reserved # -*- coding: utf-8 -*- from django import forms + from ietf.person.models import Person +from ietf.utils.fields import MultiEmailField, NameAddrEmailField class MergeForm(forms.Form): source = forms.IntegerField(label='Source Person ID') target = forms.IntegerField(label='Target Person ID') + def __init__(self, *args, **kwargs): + self.readonly = False + if 'readonly' in kwargs: + self.readonly = kwargs.pop('readonly') + super().__init__(*args, **kwargs) + if self.readonly: + self.fields['source'].widget.attrs['readonly'] = True + self.fields['target'].widget.attrs['readonly'] = True + def clean_source(self): return self.get_person(self.cleaned_data['source']) @@ -21,3 +32,11 @@ def get_person(self, pk): return Person.objects.get(pk=pk) except Person.DoesNotExist: raise forms.ValidationError("ID does not exist") + + +class MergeRequestForm(forms.Form): + to = MultiEmailField() + frm = NameAddrEmailField() + reply_to = MultiEmailField() + subject = forms.CharField() + body = forms.CharField(widget=forms.Textarea) diff --git a/ietf/person/migrations/0005_alter_historicalperson_pronouns_selectable_and_more.py b/ietf/person/migrations/0005_alter_historicalperson_pronouns_selectable_and_more.py new file mode 100644 index 0000000000..2af874b1fa --- /dev/null +++ b/ietf/person/migrations/0005_alter_historicalperson_pronouns_selectable_and_more.py @@ -0,0 +1,34 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("person", "0004_alter_person_photo_alter_person_photo_thumb"), + ] + + operations = [ + migrations.AlterField( + model_name="historicalperson", + name="pronouns_selectable", + field=models.JSONField( + blank=True, + default=list, + max_length=120, + null=True, + verbose_name="Pronouns", + ), + ), + migrations.AlterField( + model_name="person", + name="pronouns_selectable", + field=models.JSONField( + blank=True, + default=list, + max_length=120, + null=True, + verbose_name="Pronouns", + ), + ), + ] diff --git a/ietf/person/models.py b/ietf/person/models.py index 93364478ae..3ab89289a6 100644 --- a/ietf/person/models.py +++ b/ietf/person/models.py @@ -4,7 +4,6 @@ import email.utils import email.header -import jsonfield import uuid from hashids import Hashids @@ -57,7 +56,7 @@ class Person(models.Model): ascii = models.CharField("Full Name (ASCII)", max_length=255, help_text="Name as rendered in ASCII (Latin, unaccented) characters.", validators=[name_character_validator]) # The short ascii-form of the name. Also in alias table if non-null ascii_short = models.CharField("Abbreviated Name (ASCII)", max_length=32, null=True, blank=True, help_text="Example: A. Nonymous. Fill in this with initials and surname only if taking the initials and surname of the ASCII name above produces an incorrect initials-only form. (Blank is OK).", validators=[name_character_validator]) - pronouns_selectable = jsonfield.JSONCharField("Pronouns", max_length=120, blank=True, null=True, default=list ) + pronouns_selectable = models.JSONField("Pronouns", max_length=120, blank=True, null=True, default=list ) pronouns_freetext = models.CharField(" ", max_length=30, null=True, blank=True, help_text="Optionally provide your personal pronouns. These will be displayed on your public profile page and alongside your name in Meetecho and, in future, other systems. Select any number of the checkboxes OR provide a custom string up to 30 characters.") biography = models.TextField(blank=True, help_text="Short biography for use on leadership pages. Use plain text or reStructuredText markup.") photo = models.ImageField( @@ -88,7 +87,7 @@ def short(self): else: prefix, first, middle, last, suffix = self.ascii_parts() return (first and first[0]+"." or "")+(middle or "")+" "+last+(suffix and " "+suffix or "") - def plain_name(self): + def plain_name(self) -> str: if not hasattr(self, '_cached_plain_name'): if self.plain: self._cached_plain_name = self.plain @@ -204,7 +203,10 @@ def has_drafts(self): def rfcs(self): from ietf.doc.models import Document - rfcs = list(Document.objects.filter(documentauthor__person=self, type='rfc')) + # When RfcAuthors are populated, this may over-return if an author is dropped + # from the author list between the final draft and the published RFC. Should + # ignore DocumentAuthors when an RfcAuthor exists for a draft. + rfcs = list(Document.objects.filter(type="rfc").filter(models.Q(documentauthor__person=self)|models.Q(rfcauthor__person=self)).distinct()) rfcs.sort(key=lambda d: d.name ) return rfcs @@ -267,11 +269,16 @@ def available_api_endpoints(self): def cdn_photo_url(self, size=80): if self.photo: if settings.SERVE_CDN_PHOTOS: + if settings.SERVER_MODE != "production": + original_media_dir = settings.MEDIA_URL + settings.MEDIA_URL = "https://www.ietf.org/lib/dt/media/" source_url = self.photo.url if source_url.startswith(settings.IETF_HOST_URL): source_url = source_url[len(settings.IETF_HOST_URL):] elif source_url.startswith('/'): source_url = source_url[1:] + if settings.SERVER_MODE != "production": + settings.MEDIA_URL = original_media_dir return f'{settings.IETF_HOST_URL}cdn-cgi/image/fit=scale-down,width={size},height={size}/{source_url}' else: datatracker_photo_path = urlreverse('ietf.person.views.photo', kwargs={'email_or_name': self.email()}) diff --git a/ietf/person/templatetags/person_filters.py b/ietf/person/templatetags/person_filters.py index 017b29c63a..a7a6e8193a 100644 --- a/ietf/person/templatetags/person_filters.py +++ b/ietf/person/templatetags/person_filters.py @@ -50,6 +50,7 @@ def person_link(person, **kwargs): title = kwargs.get("title", "") cls = kwargs.get("class", "") with_email = kwargs.get("with_email", True) + titlepage_name = kwargs.get("titlepage_name", None) if person is not None: plain_name = person.plain_name() name = ( @@ -61,6 +62,7 @@ def person_link(person, **kwargs): return { "name": name, "plain_name": plain_name, + "titlepage_name": titlepage_name, "email": email, "title": title, "class": cls, diff --git a/ietf/person/templatetags/tests.py b/ietf/person/templatetags/tests.py index 327cfad6ce..7c35fd6b69 100644 --- a/ietf/person/templatetags/tests.py +++ b/ietf/person/templatetags/tests.py @@ -1,4 +1,6 @@ # Copyright The IETF Trust 2022, All Rights Reserved +from django.template.loader import render_to_string + from ietf.person.factories import PersonFactory from ietf.utils.test_utils import TestCase @@ -8,7 +10,6 @@ class PersonLinkTests(TestCase): # Tests of the person_link template tag. These assume it is implemented as an # inclusion tag. - # TODO test that the template actually renders the data in the dict def test_person_link(self): person = PersonFactory() self.assertEqual( @@ -16,6 +17,7 @@ def test_person_link(self): { 'name': person.name, 'plain_name': person.plain_name(), + 'titlepage_name': None, 'email': person.email_address(), 'title': '', 'class': '', @@ -27,6 +29,7 @@ def test_person_link(self): { 'name': person.name, 'plain_name': person.plain_name(), + 'titlepage_name': None, 'email': person.email_address(), 'title': '', 'class': '', @@ -38,6 +41,7 @@ def test_person_link(self): { 'name': person.name, 'plain_name': person.plain_name(), + 'titlepage_name': None, 'email': person.email_address(), 'title': 'Random Title', 'class': '', @@ -50,12 +54,71 @@ def test_person_link(self): { 'name': person.name, 'plain_name': person.plain_name(), + 'titlepage_name': None, 'email': person.email_address(), 'title': '', 'class': 'some-class', 'with_email': True, } ) + self.assertEqual( + person_link(person, titlepage_name='G. Surname'), + { + 'name': person.name, + 'plain_name': person.plain_name(), + 'titlepage_name': 'G. Surname', + 'email': person.email_address(), + 'title': '', + 'class': '', + 'with_email': True, + } + ) + + def test_person_link_renders(self): + """Verifies person/person_link.html renders context dict values correctly.""" + person = PersonFactory() + name = person.name + email = person.email_address() + base_context = { + 'name': name, + 'plain_name': person.plain_name(), + 'titlepage_name': None, + 'email': email, + 'title': '', + 'class': '', + 'with_email': True, + } + + # Default: name is used as link text with default title attribute + html = render_to_string('person/person_link.html', base_context) + self.assertIn(f'>{name}', html) + self.assertIn(f'Datatracker profile of {name}', html) + self.assertIn('bi-envelope', html) + + # titlepage_name overrides name as link text + html = render_to_string('person/person_link.html', {**base_context, 'titlepage_name': 'G. Surname'}) + self.assertIn('>G. Surname', html) + self.assertNotIn(f'>{name}', html) + + # with_email=False suppresses the envelope link + html = render_to_string('person/person_link.html', {**base_context, 'with_email': False}) + self.assertNotIn('bi-envelope', html) + + # Custom title appears in the anchor title attribute + html = render_to_string('person/person_link.html', {**base_context, 'title': 'Special Title'}) + self.assertIn('title="Special Title"', html) + + # Empty context (None person) renders (None) + self.assertInHTML( + '(None)', + render_to_string('person/person_link.html', {}), + ) + + # System email renders (System) + self.assertInHTML( + '(System)', + render_to_string('person/person_link.html', {'email': 'system@datatracker.ietf.org', 'name': ''}), + ) def test_invalid_person(self): """Generates correct context dict when input is invalid/missing""" diff --git a/ietf/person/tests.py b/ietf/person/tests.py index 61d9b0ed70..f55d8b8a34 100644 --- a/ietf/person/tests.py +++ b/ietf/person/tests.py @@ -1,16 +1,15 @@ -# Copyright The IETF Trust 2014-2022, All Rights Reserved +# Copyright The IETF Trust 2014-2025, All Rights Reserved # -*- coding: utf-8 -*- import datetime import json -import mock +from unittest import mock from io import StringIO, BytesIO from PIL import Image from pyquery import PyQuery - from django.core.exceptions import ValidationError from django.http import HttpRequest from django.test import override_settings @@ -23,6 +22,7 @@ from ietf.community.models import CommunityList from ietf.group.factories import RoleFactory from ietf.group.models import Group +from ietf.message.models import Message from ietf.nomcom.models import NomCom from ietf.nomcom.test_data import nomcom_test_data from ietf.nomcom.factories import NomComFactory, NomineeFactory, NominationFactory, FeedbackFactory, PositionFactory @@ -208,13 +208,13 @@ def test_merge(self): def test_merge_with_params(self): p1 = get_person_no_user() p2 = PersonFactory() - url = urlreverse("ietf.person.views.merge") + "?source={}&target={}".format(p1.pk, p2.pk) + url = urlreverse("ietf.person.views.merge_submit") + "?source={}&target={}".format(p1.pk, p2.pk) login_testing_unauthorized(self, "secretary", url) r = self.client.get(url) self.assertContains(r, 'retaining login', status_code=200) def test_merge_with_params_bad_id(self): - url = urlreverse("ietf.person.views.merge") + "?source=1000&target=2000" + url = urlreverse("ietf.person.views.merge_submit") + "?source=1000&target=2000" login_testing_unauthorized(self, "secretary", url) r = self.client.get(url) self.assertContains(r, 'ID does not exist', status_code=200) @@ -222,7 +222,7 @@ def test_merge_with_params_bad_id(self): def test_merge_post(self): p1 = get_person_no_user() p2 = PersonFactory() - url = urlreverse("ietf.person.views.merge") + url = urlreverse("ietf.person.views.merge_submit") expected_url = urlreverse("ietf.secr.rolodex.views.view", kwargs={'id': p2.pk}) login_testing_unauthorized(self, "secretary", url) data = {'source': p1.pk, 'target': p2.pk} @@ -451,6 +451,30 @@ def test_dots(self): ncchair = RoleFactory(group__acronym='nomcom2020',group__type_id='nomcom',name_id='chair').person self.assertEqual(get_dots(ncchair),['nomcom']) + def test_send_merge_request(self): + empty_outbox() + message_count_before = Message.objects.count() + source = PersonFactory() + target = PersonFactory() + url = urlreverse('ietf.person.views.send_merge_request') + url = url + f'?source={source.pk}&target={target.pk}' + login_testing_unauthorized(self, 'secretary', url) + r = self.client.get(url) + initial = r.context['form'].initial + subject = 'Action requested: Merging possible duplicate IETF Datatracker accounts' + self.assertEqual(initial['to'], ', '.join([source.user.username, target.user.username])) + self.assertEqual(initial['subject'], subject) + self.assertEqual(initial['reply_to'], 'support@ietf.org') + self.assertEqual(r.status_code, 200) + r = self.client.post(url, data=initial) + self.assertEqual(r.status_code, 302) + self.assertEqual(len(outbox), 1) + self.assertIn(source.user.username, outbox[0]['To']) + message_count_after = Message.objects.count() + message = Message.objects.last() + self.assertEqual(message_count_after, message_count_before + 1) + self.assertIn(source.user.username, message.to) + class TaskTests(TestCase): @mock.patch("ietf.person.tasks.log.log") diff --git a/ietf/person/urls.py b/ietf/person/urls.py index 867646fe39..f3eccd04b7 100644 --- a/ietf/person/urls.py +++ b/ietf/person/urls.py @@ -1,8 +1,12 @@ +# Copyright The IETF Trust 2009-2025, All Rights Reserved +# -*- coding: utf-8 -*- from ietf.person import views, ajax from ietf.utils.urls import url urlpatterns = [ url(r'^merge/?$', views.merge), + url(r'^merge/submit/?$', views.merge_submit), + url(r'^merge/send_request/?$', views.send_merge_request), url(r'^search/(?P(person|email))/$', views.ajax_select2_search), url(r'^(?P[0-9]+)/email.json$', ajax.person_email_json), url(r'^(?P[^/]+)$', views.profile), diff --git a/ietf/person/views.py b/ietf/person/views.py index bb1fa79f84..d0b5912431 100644 --- a/ietf/person/views.py +++ b/ietf/person/views.py @@ -1,14 +1,16 @@ -# Copyright The IETF Trust 2012-2020, All Rights Reserved +# Copyright The IETF Trust 2012-2025, All Rights Reserved # -*- coding: utf-8 -*- from io import StringIO, BytesIO from PIL import Image +from django.conf import settings from django.contrib import messages from django.db.models import Q from django.http import HttpResponse, Http404 from django.shortcuts import render, redirect +from django.template.loader import render_to_string from django.utils import timezone import debug # pyflakes:ignore @@ -16,8 +18,9 @@ from ietf.ietfauth.utils import role_required from ietf.person.models import Email, Person from ietf.person.fields import select2_id_name_json -from ietf.person.forms import MergeForm +from ietf.person.forms import MergeForm, MergeRequestForm from ietf.person.utils import handle_users, merge_persons, lookup_persons +from ietf.utils.mail import send_mail_text def ajax_select2_search(request, model_name): @@ -62,8 +65,9 @@ def ajax_select2_search(request, model_name): page = int(request.GET.get("p", 1)) - 1 except ValueError: page = 0 - - objs = objs.distinct()[page:page + 10] + PAGE_SIZE = 10 + first_item = page * PAGE_SIZE + objs = objs.distinct()[first_item:first_item + PAGE_SIZE] return HttpResponse(select2_id_name_json(objs), content_type='application/json') @@ -97,16 +101,19 @@ def photo(request, email_or_name): @role_required("Secretariat") def merge(request): form = MergeForm() - method = 'get' + return render(request, 'person/merge.html', {'form': form}) + + +@role_required("Secretariat") +def merge_submit(request): change_details = '' warn_messages = [] source = None target = None if request.method == "GET": - form = MergeForm() if request.GET: - form = MergeForm(request.GET) + form = MergeForm(request.GET, readonly=True) if form.is_valid(): source = form.cleaned_data.get('source') target = form.cleaned_data.get('target') @@ -115,12 +122,9 @@ def merge(request): if source.user.last_login and target.user.last_login and source.user.last_login > target.user.last_login: warn_messages.append('WARNING: The most recently used login is being deleted!') change_details = handle_users(source, target, check_only=True) - method = 'post' - else: - method = 'get' if request.method == "POST": - form = MergeForm(request.POST) + form = MergeForm(request.POST, readonly=True) if form.is_valid(): source = form.cleaned_data.get('source') source_id = source.id @@ -135,11 +139,72 @@ def merge(request): messages.error(request, output) return redirect('ietf.secr.rolodex.views.view', id=target.pk) - return render(request, 'person/merge.html', { + return render(request, 'person/merge_submit.html', { 'form': form, - 'method': method, 'change_details': change_details, 'source': source, 'target': target, 'warn_messages': warn_messages, }) + + +@role_required("Secretariat") +def send_merge_request(request): + if request.method == 'GET': + merge_form = MergeForm(request.GET) + if merge_form.is_valid(): + source = merge_form.cleaned_data['source'] + target = merge_form.cleaned_data['target'] + to = [] + if source.email(): + to.append(source.email().address) + if target.email(): + to.append(target.email().address) + if source.user: + source_account = source.user.username + else: + source_account = source.email() + if target.user: + target_account = target.user.username + else: + target_account = target.email() + sender_name = request.user.person.name + subject = 'Action requested: Merging possible duplicate IETF Datatracker accounts' + context = { + 'source_account': source_account, + 'target_account': target_account, + 'sender_name': sender_name, + } + body = render_to_string('person/merge_request_email.txt', context) + initial = { + 'to': ', '.join(to), + 'frm': settings.DEFAULT_FROM_EMAIL, + 'reply_to': 'support@ietf.org', + 'subject': subject, + 'body': body, + 'by': request.user.person.pk, + } + form = MergeRequestForm(initial=initial) + else: + messages.error(request, "Error requesting merge email: " + merge_form.errors.as_text()) + return redirect("ietf.person.views.merge") + + if request.method == 'POST': + form = MergeRequestForm(request.POST) + if form.is_valid(): + extra = {"Reply-To": form.cleaned_data.get("reply_to")} + send_mail_text( + request, + form.cleaned_data.get("to"), + form.cleaned_data.get("frm"), + form.cleaned_data.get("subject"), + form.cleaned_data.get("body"), + extra=extra, + ) + + messages.success(request, "The merge confirmation email was sent.") + return redirect("ietf.person.views.merge") + + return render(request, "person/send_merge_request.html", { + "form": form, + }) diff --git a/ietf/review/mailarch.py b/ietf/review/mailarch.py index c34a6079ce..61abc83aa5 100644 --- a/ietf/review/mailarch.py +++ b/ietf/review/mailarch.py @@ -6,25 +6,18 @@ # mailarchive.ietf.org import base64 -import contextlib import datetime import email.utils import hashlib -import mailbox -import tarfile -import tempfile - -from urllib.parse import urlencode -from urllib.request import urlopen +import requests import debug # pyflakes:ignore -from pyquery import PyQuery from django.conf import settings from django.utils.encoding import force_bytes, force_str -from ietf.utils.mail import get_payload_text +from ietf.utils.log import log from ietf.utils.timezone import date_today @@ -43,7 +36,7 @@ def hash_list_message_id(list_name, msgid): sha.update(force_bytes(list_name)) return force_str(base64.urlsafe_b64encode(sha.digest()).rstrip(b"=")) -def construct_query_urls(doc, team, query=None): +def construct_query_data(doc, team, query=None): list_name = list_name_from_email(team.list_email) if not list_name: return None @@ -51,83 +44,48 @@ def construct_query_urls(doc, team, query=None): if not query: query = doc.name - encoded_query = "?" + urlencode({ - "qdr": "c", # custom time frame - "start_date": (date_today() - datetime.timedelta(days=180)).isoformat(), - "email_list": list_name, - "q": "subject:({})".format(query), - "as": "1", # this is an advanced search - }) - - return { - "query": query, - "query_url": settings.MAILING_LIST_ARCHIVE_URL + "/arch/search/" + encoded_query, - "query_data_url": settings.MAILING_LIST_ARCHIVE_URL + "/arch/export/mbox/" + encoded_query, + query_data = { + 'start_date': (date_today() - datetime.timedelta(days=180)).isoformat(), + 'email_list': list_name, + 'query_value': query, + 'query': f'subject:({query})', + 'limit': '30', } + return query_data def construct_message_url(list_name, msgid): return "{}/arch/msg/{}/{}".format(settings.MAILING_LIST_ARCHIVE_URL, list_name, hash_list_message_id(list_name, msgid)) -def retrieve_messages_from_mbox(mbox_fileobj): - """Return selected content in message from mbox from mailarch.""" - res = [] - with tempfile.NamedTemporaryFile(suffix=".mbox") as mbox_file: - # mailbox.mbox needs a path, so we need to put the contents - # into a file - mbox_data = mbox_fileobj.read() - mbox_file.write(mbox_data) - mbox_file.flush() - - mbox = mailbox.mbox(mbox_file.name, create=False) - for msg in mbox: - content = "" - - for part in msg.walk(): - if part.get_content_type() == "text/plain": - charset = part.get_content_charset() or "utf-8" - content += get_payload_text(part, default_charset=charset) - - # parse a couple of things for the front end - utcdate = None - d = email.utils.parsedate_tz(msg["Date"]) - if d: - utcdate = datetime.datetime.fromtimestamp(email.utils.mktime_tz(d), datetime.timezone.utc) - - res.append({ - "from": msg["From"], - "splitfrom": email.utils.parseaddr(msg["From"]), - "subject": msg["Subject"], - "content": content.replace("\r\n", "\n").replace("\r", "\n").strip("\n"), - "message_id": email.utils.unquote(msg["Message-ID"].strip()), - "url": email.utils.unquote(msg["Archived-At"].strip()), - "date": msg["Date"], - "utcdate": (utcdate.date().isoformat(), utcdate.time().isoformat()) if utcdate else ("", ""), - }) - mbox.close() - - return res - -def retrieve_messages(query_data_url): +def retrieve_messages(query_data): """Retrieve and return selected content from mailarch.""" - res = [] - - # This has not been rewritten to use requests.get() because get() does - # not handle file URLs out of the box, which we need for tesing - with contextlib.closing(urlopen(query_data_url, timeout=15)) as fileobj: - content_type = fileobj.info()["Content-type"] - if not content_type.startswith("application/x-tar"): - if content_type.startswith("text/html"): - r = fileobj.read(20000) - q = PyQuery(r) - div = q('div[class~="no-results"]') - if div: - raise KeyError("No results: %s -> %s" % (query_data_url, div.text(), )) - raise Exception("Export failed - this usually means no matches were found") - - with tarfile.open(fileobj=fileobj, mode='r|*') as tar: - for entry in tar: - if entry.isfile(): - mbox_fileobj = tar.extractfile(entry) - res.extend(retrieve_messages_from_mbox(mbox_fileobj)) - - return res + + headers = {'X-Api-Key': settings.MAILING_LIST_ARCHIVE_API_KEY} + try: + response = requests.post( + settings.MAILING_LIST_ARCHIVE_SEARCH_URL, + headers=headers, + json=query_data, + timeout=settings.DEFAULT_REQUESTS_TIMEOUT) + except requests.Timeout as exc: + log(f'POST request failed for [{query_data["url"]}]: {exc}') + raise RuntimeError(f'Timeout retrieving [{query_data["url"]}]') from exc + + results = [] + jresponse = response.json() + if 'results' not in jresponse or len(jresponse['results']) == 0: + raise KeyError(f'No results: {query_data["query"]}') + for msg in jresponse['results']: + # datetime is already UTC + dt = datetime.datetime.fromisoformat(msg['date']) + dt_utc = dt.replace(tzinfo=datetime.timezone.utc) + results.append({ + "from": msg["from"], + "splitfrom": email.utils.parseaddr(msg["from"]), + "subject": msg["subject"], + "content": msg["content"].replace("\r\n", "\n").replace("\r", "\n").strip("\n"), + "message_id": msg["message_id"], + "url": msg["url"], + "utcdate": (dt_utc.date().isoformat(), dt_utc.time().isoformat()), + }) + + return results diff --git a/ietf/review/tests.py b/ietf/review/tests.py index e9ddbd47af..5dc8f11e8e 100644 --- a/ietf/review/tests.py +++ b/ietf/review/tests.py @@ -1,7 +1,7 @@ # Copyright The IETF Trust 2019-2020, All Rights Reserved # -*- coding: utf-8 -*- import datetime -import mock +from unittest import mock import debug # pyflakes:ignore from pyquery import PyQuery diff --git a/ietf/secr/announcement/forms.py b/ietf/secr/announcement/forms.py index 3fe58bdaaa..91004ea270 100644 --- a/ietf/secr/announcement/forms.py +++ b/ietf/secr/announcement/forms.py @@ -14,30 +14,41 @@ # Globals # --------------------------------------------- -TO_LIST = ('IETF Announcement List ', - 'I-D Announcement List ', - 'RFP Announcement List ', - 'The IESG ', - 'Working Group Chairs ', - 'BOF Chairs ', - 'Other...') +TO_LIST = ( + "IETF Announcement List ", + "I-D Announcement List ", + "RFP Announcement List ", + "The IESG ", + "Working Group Chairs ", + "BOF Chairs ", + "Other...", +) # --------------------------------------------- # Helper Functions # --------------------------------------------- + def get_from_choices(user): - ''' + """ This function returns a choices tuple containing all the Announced From choices. Including leadership chairs and other entities. - ''' + """ addresses = [] - if has_role(user,'Secretariat'): - addresses = AnnouncementFrom.objects.values_list('address', flat=True).order_by('address').distinct() + if has_role(user, "Secretariat"): + addresses = ( + AnnouncementFrom.objects.values_list("address", flat=True) + .order_by("address") + .distinct() + ) else: for role in user.person.role_set.all(): - addresses.extend(AnnouncementFrom.objects.filter(name=role.name, group=role.group).values_list('address', flat=True).order_by('address')) + addresses.extend( + AnnouncementFrom.objects.filter(name=role.name, group=role.group) + .values_list("address", flat=True) + .order_by("address") + ) nomcom_choices = get_nomcom_choices(user) if nomcom_choices: @@ -45,65 +56,91 @@ def get_from_choices(user): choices = list(zip(addresses, addresses)) if len(choices) > 1: - choices.insert(0, ('', '(Choose an option)')) + choices.insert(0, ("", "(Choose an option)")) return choices def get_nomcom_choices(user): - ''' + """ Returns the list of nomcom email addresses for given user - ''' - nomcoms = Role.objects.filter(name="chair", - group__acronym__startswith="nomcom", - group__state="active", - group__type="nomcom", - person=user.person) + """ + nomcoms = Role.objects.filter( + name="chair", + group__acronym__startswith="nomcom", + group__state="active", + group__type="nomcom", + person=user.person, + ) addresses = [] for nomcom in nomcoms: year = nomcom.group.acronym[-4:] - addresses.append('NomCom Chair %s ' % (year,year)) + addresses.append("NomCom Chair %s " % (year, year)) return addresses - + def get_to_choices(): - return list(zip(TO_LIST,TO_LIST)) + return list(zip(TO_LIST, TO_LIST)) # --------------------------------------------- # Forms # --------------------------------------------- + class AnnounceForm(forms.ModelForm): - nomcom = forms.ModelChoiceField(queryset=Group.objects.filter(acronym__startswith='nomcom',type='nomcom',state='active'),required=False) + nomcom = forms.ModelChoiceField( + queryset=Group.objects.filter( + acronym__startswith="nomcom", type="nomcom", state="active" + ), + required=False, + ) to_custom = MultiEmailField(required=False) class Meta: model = Message - fields = ('nomcom', 'to','to_custom','frm','cc','bcc','reply_to','subject','body') + fields = ( + "nomcom", + "to", + "to_custom", + "frm", + "cc", + "bcc", + "reply_to", + "subject", + "body", + ) + labels = {"frm": "From"} + help_texts = { + "to": "Select name OR select Other... and enter email below", + "cc": "Use comma separated lists for emails (Cc, Bcc, Reply To)", + } def __init__(self, *args, **kwargs): - if 'hidden' in kwargs: - self.hidden = kwargs.pop('hidden') + if "hidden" in kwargs: + self.hidden = kwargs.pop("hidden") else: self.hidden = False - user = kwargs.pop('user') + user = kwargs.pop("user") person = user.person super(AnnounceForm, self).__init__(*args, **kwargs) - self.fields['to'].widget = forms.Select(choices=get_to_choices()) - self.fields['to'].help_text = 'Select name OR select Other... and enter email below' - self.fields['cc'].help_text = 'Use comma separated lists for emails (Cc, Bcc, Reply To)' - self.fields['frm'].widget = forms.Select(choices=get_from_choices(user)) - self.fields['frm'].label = 'From' - self.fields['reply_to'].required = True - self.fields['nomcom'].label = 'NomCom message:' - nomcom_roles = person.role_set.filter(group__in=self.fields['nomcom'].queryset,name='chair') - secr_roles = person.role_set.filter(group__acronym='secretariat',name='secr') + self.fields["to"].widget = forms.Select(choices=get_to_choices()) + self.fields["frm"].widget = forms.Select(choices=get_from_choices(user)) + self.fields["reply_to"].required = True + # nomcom field is defined declaratively so label and help_text must be set here + self.fields["nomcom"].label = "NomCom message:" + self.fields["nomcom"].help_text = ( + "If this is a NomCom announcement specify which NomCom group here" + ) + nomcom_roles = person.role_set.filter( + group__in=self.fields["nomcom"].queryset, name="chair" + ) + secr_roles = person.role_set.filter(group__acronym="secretariat", name="secr") if nomcom_roles: - self.initial['nomcom'] = nomcom_roles[0].group.pk + self.initial["nomcom"] = nomcom_roles[0].group.pk if not nomcom_roles and not secr_roles: - self.fields['nomcom'].widget = forms.HiddenInput() - + self.fields["nomcom"].widget = forms.HiddenInput() + if self.hidden: for key in list(self.fields.keys()): self.fields[key].widget = forms.HiddenInput() @@ -113,25 +150,29 @@ def clean(self): data = self.cleaned_data if self.errors: return self.cleaned_data - if data['to'] == 'Other...' and not data['to_custom']: + if data["to"] == "Other..." and not data["to_custom"]: raise forms.ValidationError('You must enter a "To" email address') - for k in ['to', 'frm', 'cc',]: + for k in [ + "to", + "frm", + "cc", + ]: data[k] = unescape(data[k]) return data def save(self, *args, **kwargs): - user = kwargs.pop('user') + user = kwargs.pop("user") message = super(AnnounceForm, self).save(commit=False) message.by = user.person - if self.cleaned_data['to'] == 'Other...': - message.to = self.cleaned_data['to_custom'] - if kwargs['commit']: + if self.cleaned_data["to"] == "Other...": + message.to = self.cleaned_data["to_custom"] + if kwargs["commit"]: message.save() # handle nomcom message - nomcom = self.cleaned_data.get('nomcom',False) + nomcom = self.cleaned_data.get("nomcom", False) if nomcom: message.related_groups.add(nomcom) - return message \ No newline at end of file + return message diff --git a/ietf/secr/announcement/tests.py b/ietf/secr/announcement/tests.py index c147c301b6..f08e824397 100644 --- a/ietf/secr/announcement/tests.py +++ b/ietf/secr/announcement/tests.py @@ -6,7 +6,7 @@ from django.urls import reverse -import debug # pyflakes:ignore +import debug # pyflakes:ignore from ietf.utils.test_utils import TestCase from ietf.group.factories import RoleFactory @@ -17,97 +17,102 @@ from ietf.message.models import AnnouncementFrom from ietf.utils.mail import outbox, empty_outbox -SECR_USER='secretary' -WG_USER='' -AD_USER='' +SECR_USER = "secretary" +WG_USER = "" +AD_USER = "" + class SecrAnnouncementTestCase(TestCase): def setUp(self): super().setUp() - chair = RoleName.objects.get(slug='chair') - secr = RoleName.objects.get(slug='secr') - ietf = Group.objects.get(acronym='ietf') - iab = Group.objects.get(acronym='iab') - secretariat = Group.objects.get(acronym='secretariat') - AnnouncementFrom.objects.create(name=secr,group=secretariat,address='IETF Secretariat ') - AnnouncementFrom.objects.create(name=chair,group=ietf,address='IETF Chair ') - AnnouncementFrom.objects.create(name=chair,group=iab,address='IAB Chair ') + chair = RoleName.objects.get(slug="chair") + secr = RoleName.objects.get(slug="secr") + ietf = Group.objects.get(acronym="ietf") + iab = Group.objects.get(acronym="iab") + secretariat = Group.objects.get(acronym="secretariat") + AnnouncementFrom.objects.create( + name=secr, + group=secretariat, + address="IETF Secretariat ", + ) + AnnouncementFrom.objects.create( + name=chair, group=ietf, address="IETF Chair " + ) + AnnouncementFrom.objects.create( + name=chair, group=iab, address="IAB Chair " + ) def test_main(self): "Main Test" - url = reverse('ietf.secr.announcement.views.main') + url = reverse("ietf.secr.announcement.views.main") self.client.login(username="secretary", password="secretary+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) - + def test_main_announce_from(self): - url = reverse('ietf.secr.announcement.views.main') + url = reverse("ietf.secr.announcement.views.main") # Secretariat self.client.login(username="secretary", password="secretary+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertEqual(len(q('#id_frm option')),4) + self.assertEqual(len(q("#id_frm option")), 4) # IAB Chair self.client.login(username="iab-chair", password="iab-chair+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertEqual(len(q('#id_frm option')),1) - self.assertTrue('' in q('#id_frm option').val()) + self.assertEqual(len(q("#id_frm option")), 1) + self.assertTrue("" in q("#id_frm option").val()) # IETF Chair self.client.login(username="ietf-chair", password="ietf-chair+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertEqual(len(q('#id_frm option')),1) - self.assertTrue('' in q('#id_frm option').val()) + self.assertEqual(len(q("#id_frm option")), 1) + self.assertTrue("" in q("#id_frm option").val()) + class UnauthorizedAnnouncementCase(TestCase): def test_unauthorized(self): "Unauthorized Test" - url = reverse('ietf.secr.announcement.views.main') - person = RoleFactory(name_id='chair',group__acronym='mars').person - self.client.login(username=person.user.username, password=person.user.username+"+password") + url = reverse("ietf.secr.announcement.views.main") + person = RoleFactory(name_id="chair", group__acronym="mars").person + self.client.login( + username=person.user.username, password=person.user.username + "+password" + ) r = self.client.get(url) self.assertEqual(r.status_code, 403) - + + class SubmitAnnouncementCase(TestCase): - def test_invalid_submit(self): - "Invalid Submit" - url = reverse('ietf.secr.announcement.views.main') - post_data = {'id_subject':''} - self.client.login(username="secretary", password="secretary+password") - r = self.client.post(url,post_data) - self.assertEqual(r.status_code, 200) - q = PyQuery(r.content) - self.assertTrue(len(q('form ul.errorlist')) > 0) - def test_valid_submit(self): "Valid Submit" nomcom_test_data() empty_outbox() - url = reverse('ietf.secr.announcement.views.main') - confirm_url = reverse('ietf.secr.announcement.views.confirm') - nomcom = Group.objects.get(type='nomcom') - post_data = {'nomcom': nomcom.pk, - 'to':'Other...', - 'to_custom':'rcross@amsl.com', - 'frm':'IETF Secretariat <ietf-secretariat@ietf.org>', - 'reply_to':'secretariat@ietf.org', - 'subject':'Test Subject', - 'body':'This is a test.'} + url = reverse("ietf.secr.announcement.views.main") + confirm_url = reverse("ietf.secr.announcement.views.confirm") + nomcom = Group.objects.get(type="nomcom") + post_data = { + "nomcom": nomcom.pk, + "to": "Other...", + "to_custom": "phil@example.com", + "frm": "IETF Secretariat <ietf-secretariat@ietf.org>", + "reply_to": "secretariat@ietf.org", + "subject": "Test Subject", + "body": "This is a test.", + } self.client.login(username="secretary", password="secretary+password") - response = self.client.post(url,post_data) - self.assertContains(response, 'Confirm Announcement') - response = self.client.post(confirm_url,post_data,follow=True) + response = self.client.post(url, post_data) + self.assertContains(response, "Confirm Announcement") + response = self.client.post(confirm_url, post_data, follow=True) self.assertRedirects(response, url) - self.assertEqual(len(outbox),1) - self.assertEqual(outbox[0]['subject'],'Test Subject') - self.assertEqual(outbox[0]['to'],'') - message = Message.objects.filter(by__user__username='secretary').last() - self.assertEqual(message.subject,'Test Subject') + self.assertEqual(len(outbox), 1) + self.assertEqual(outbox[0]["subject"], "Test Subject") + self.assertEqual(outbox[0]["to"], "") + message = Message.objects.filter(by__user__username="secretary").last() + self.assertEqual(message.subject, "Test Subject") self.assertTrue(nomcom in message.related_groups.all()) diff --git a/ietf/secr/announcement/urls.py b/ietf/secr/announcement/urls.py index 3c3c05a09c..dc534f64ae 100644 --- a/ietf/secr/announcement/urls.py +++ b/ietf/secr/announcement/urls.py @@ -1,8 +1,7 @@ - from ietf.secr.announcement import views from ietf.utils.urls import url urlpatterns = [ - url(r'^$', views.main), - url(r'^confirm/$', views.confirm), + url(r"^$", views.main), + url(r"^confirm/$", views.confirm), ] diff --git a/ietf/secr/announcement/views.py b/ietf/secr/announcement/views.py index 42de089c59..5617ae9e6f 100644 --- a/ietf/secr/announcement/views.py +++ b/ietf/secr/announcement/views.py @@ -18,86 +18,93 @@ # Helper Functions # ------------------------------------------------- def check_access(user): - ''' + """ This function takes a Django User object and returns true if the user has access to the Announcement app. - ''' + """ if hasattr(user, "person"): person = user.person if has_role(user, "Secretariat"): return True - + for role in person.role_set.all(): - if AnnouncementFrom.objects.filter(name=role.name,group=role.group): + if AnnouncementFrom.objects.filter(name=role.name, group=role.group): return True - if Role.objects.filter(name="chair", - group__acronym__startswith="nomcom", - group__state="active", - group__type="nomcom", - person=person): + if Role.objects.filter( + name="chair", + group__acronym__startswith="nomcom", + group__state="active", + group__type="nomcom", + person=person, + ): return True return False + # -------------------------------------------------- # STANDARD VIEW FUNCTIONS # -------------------------------------------------- # this seems to cause some kind of circular problem # @check_for_cancel(reverse('home')) @login_required -@check_for_cancel('../') +@check_for_cancel("../") def main(request): - ''' + """ Main view for Announcement tool. Authrozied users can fill out email details: header, body, etc and send. - ''' + """ if not check_access(request.user): - permission_denied(request, 'Restricted to: Secretariat, IAD, or chair of IETF, IAB, RSOC, RSE, IAOC, ISOC, NomCom.') + permission_denied( + request, + "Restricted to: Secretariat, IAD, or chair of IETF, IAB, RSOC, RSE, IAOC, ISOC, NomCom.", + ) - form = AnnounceForm(request.POST or None,user=request.user) + form = AnnounceForm(request.POST or None, user=request.user) if form.is_valid(): # recast as hidden form for next page of process form = AnnounceForm(request.POST, user=request.user, hidden=True) - if form.data['to'] == 'Other...': - to = form.data['to_custom'] + if form.data["to"] == "Other...": + to = form.data["to_custom"] else: - to = form.data['to'] + to = form.data["to"] - return render(request, 'announcement/confirm.html', { - 'message': form.data, - 'to': to, - 'form': form}, + return render( + request, + "announcement/confirm.html", + {"message": form.data, "to": to, "form": form}, ) - return render(request, 'announcement/main.html', { 'form': form} ) + return render(request, "announcement/index.html", {"form": form}) + @login_required -@check_for_cancel('../') +@check_for_cancel("../") def confirm(request): if not check_access(request.user): - permission_denied(request, 'Restricted to: Secretariat, IAD, or chair of IETF, IAB, RSOC, RSE, IAOC, ISOC, NomCom.') + permission_denied( + request, + "Restricted to: Secretariat, IAD, or chair of IETF, IAB, RSOC, RSE, IAOC, ISOC, NomCom.", + ) - if request.method == 'POST': + if request.method == "POST": form = AnnounceForm(request.POST, user=request.user) - if request.method == 'POST': - message = form.save(user=request.user,commit=True) - extra = {'Reply-To': message.get('reply_to') } - send_mail_text(None, - message.to, - message.frm, - message.subject, - message.body, - cc=message.cc, - bcc=message.bcc, - extra=extra, - ) - - messages.success(request, 'The announcement was sent.') - return redirect('ietf.secr.announcement.views.main') - - - - + if request.method == "POST": + message = form.save(user=request.user, commit=True) + extra = {"Reply-To": message.get("reply_to")} + send_mail_text( + None, + message.to, + message.frm, + message.subject, + message.body, + cc=message.cc, + bcc=message.bcc, + extra=extra, + ) + + messages.success(request, "The announcement was sent.") + return redirect("ietf.secr.announcement.views.main") diff --git a/ietf/secr/meetings/views.py b/ietf/secr/meetings/views.py index 47f7b7ffa5..1f6f2f3297 100644 --- a/ietf/secr/meetings/views.py +++ b/ietf/secr/meetings/views.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2007-2023, All Rights Reserved +# Copyright The IETF Trust 2007-2025, All Rights Reserved # -*- coding: utf-8 -*- import datetime @@ -20,12 +20,12 @@ from ietf.meeting.helpers import make_materials_directories, populate_important_dates from ietf.meeting.models import Meeting, Session, Room, TimeSlot, SchedTimeSessAssignment, Schedule, SchedulingEvent from ietf.meeting.utils import add_event_info_to_session_qs +from ietf.meeting.views_session_request import get_initial_session from ietf.name.models import SessionStatusName from ietf.group.models import Group, GroupEvent from ietf.secr.meetings.forms import ( BaseMeetingRoomFormSet, MeetingModelForm, MeetingSelectForm, MeetingRoomForm, MiscSessionForm, TimeSlotForm, RegularSessionEditForm, MeetingRoomOptionsForm ) -from ietf.secr.sreq.views import get_initial_session from ietf.secr.utils.meeting import get_session, get_timeslot from ietf.mailtrigger.utils import gather_address_lists from ietf.utils.timezone import make_aware diff --git a/ietf/secr/sreq/forms.py b/ietf/secr/sreq/forms.py deleted file mode 100644 index 4a0f449b2a..0000000000 --- a/ietf/secr/sreq/forms.py +++ /dev/null @@ -1,333 +0,0 @@ -# Copyright The IETF Trust 2013-2022, All Rights Reserved -# -*- coding: utf-8 -*- - - -from django import forms -from django.template.defaultfilters import pluralize - -import debug # pyflakes:ignore - -from ietf.name.models import TimerangeName, ConstraintName -from ietf.group.models import Group -from ietf.meeting.forms import sessiondetailsformset_factory -from ietf.meeting.models import ResourceAssociation, Constraint -from ietf.person.fields import SearchablePersonsField -from ietf.person.models import Person -from ietf.utils.fields import ModelMultipleChoiceField -from ietf.utils.html import clean_text_field -from ietf.utils import log - -# ------------------------------------------------- -# Globals -# ------------------------------------------------- - -NUM_SESSION_CHOICES = (('','--Please select'),('1','1'),('2','2')) -SESSION_TIME_RELATION_CHOICES = (('', 'No preference'),) + Constraint.TIME_RELATION_CHOICES -JOINT_FOR_SESSION_CHOICES = (('1', 'First session'), ('2', 'Second session'), ('3', 'Third session'), ) - -# ------------------------------------------------- -# Helper Functions -# ------------------------------------------------- -def allowed_conflicting_groups(): - return Group.objects.filter(type__in=['wg', 'ag', 'rg', 'rag', 'program', 'edwg'], state__in=['bof', 'proposed', 'active']) - -def check_conflict(groups, source_group): - ''' - Takes a string which is a list of group acronyms. Checks that they are all active groups - ''' - # convert to python list (allow space or comma separated lists) - items = groups.replace(',',' ').split() - active_groups = allowed_conflicting_groups() - for group in items: - if group == source_group.acronym: - raise forms.ValidationError("Cannot declare a conflict with the same group: %s" % group) - - if not active_groups.filter(acronym=group): - raise forms.ValidationError("Invalid or inactive group acronym: %s" % group) - -# ------------------------------------------------- -# Forms -# ------------------------------------------------- - -class GroupSelectForm(forms.Form): - group = forms.ChoiceField() - - def __init__(self,*args,**kwargs): - choices = kwargs.pop('choices') - super(GroupSelectForm, self).__init__(*args,**kwargs) - self.fields['group'].widget.choices = choices - - -class NameModelMultipleChoiceField(ModelMultipleChoiceField): - def label_from_instance(self, name): - return name.desc - - -class SessionForm(forms.Form): - num_session = forms.ChoiceField(choices=NUM_SESSION_CHOICES) - # session fields are added in __init__() - session_time_relation = forms.ChoiceField(choices=SESSION_TIME_RELATION_CHOICES, required=False) - attendees = forms.IntegerField() - # FIXME: it would cleaner to have these be - # ModelMultipleChoiceField, and just customize the widgetry, that - # way validation comes for free (applies to this CharField and the - # constraints dynamically instantiated in __init__()) - joint_with_groups = forms.CharField(max_length=255,required=False) - joint_with_groups_selector = forms.ChoiceField(choices=[], required=False) # group select widget for prev field - joint_for_session = forms.ChoiceField(choices=JOINT_FOR_SESSION_CHOICES, required=False) - comments = forms.CharField(max_length=200,required=False) - third_session = forms.BooleanField(required=False) - resources = forms.MultipleChoiceField(widget=forms.CheckboxSelectMultiple,required=False) - bethere = SearchablePersonsField(label="Must be present", required=False) - timeranges = NameModelMultipleChoiceField(widget=forms.CheckboxSelectMultiple, required=False, - queryset=TimerangeName.objects.all()) - adjacent_with_wg = forms.ChoiceField(required=False) - send_notifications = forms.BooleanField(label="Send notification emails?", required=False, initial=False) - - def __init__(self, group, meeting, data=None, *args, **kwargs): - self.hidden = kwargs.pop('hidden', False) - self.notifications_optional = kwargs.pop('notifications_optional', False) - - self.group = group - formset_class = sessiondetailsformset_factory(max_num=3 if group.features.acts_like_wg else 50) - self.session_forms = formset_class(group=self.group, meeting=meeting, data=data) - super(SessionForm, self).__init__(data=data, *args, **kwargs) - if not self.notifications_optional: - self.fields['send_notifications'].widget = forms.HiddenInput() - - # Allow additional sessions for non-wg-like groups - if not self.group.features.acts_like_wg: - self.fields['num_session'].choices = ((n, str(n)) for n in range(1, 51)) - - self.fields['comments'].widget = forms.Textarea(attrs={'rows':'3','cols':'65'}) - - other_groups = list(allowed_conflicting_groups().exclude(pk=group.pk).values_list('acronym', 'acronym').order_by('acronym')) - self.fields['adjacent_with_wg'].choices = [('', '--No preference')] + other_groups - group_acronym_choices = [('','--Select WG(s)')] + other_groups - self.fields['joint_with_groups_selector'].choices = group_acronym_choices - - # Set up constraints for the meeting - self._wg_field_data = [] - for constraintname in meeting.group_conflict_types.all(): - # two fields for each constraint: a CharField for the group list and a selector to add entries - constraint_field = forms.CharField(max_length=255, required=False) - constraint_field.widget.attrs['data-slug'] = constraintname.slug - constraint_field.widget.attrs['data-constraint-name'] = str(constraintname).title() - self._add_widget_class(constraint_field.widget, 'wg_constraint') - - selector_field = forms.ChoiceField(choices=group_acronym_choices, required=False) - selector_field.widget.attrs['data-slug'] = constraintname.slug # used by onchange handler - self._add_widget_class(selector_field.widget, 'wg_constraint_selector') - - cfield_id = 'constraint_{}'.format(constraintname.slug) - cselector_id = 'wg_selector_{}'.format(constraintname.slug) - # keep an eye out for field name conflicts - log.assertion('cfield_id not in self.fields') - log.assertion('cselector_id not in self.fields') - self.fields[cfield_id] = constraint_field - self.fields[cselector_id] = selector_field - self._wg_field_data.append((constraintname, cfield_id, cselector_id)) - - # Show constraints that are not actually used by the meeting so these don't get lost - self._inactive_wg_field_data = [] - inactive_cnames = ConstraintName.objects.filter( - is_group_conflict=True # Only collect group conflicts... - ).exclude( - meeting=meeting # ...that are not enabled for this meeting... - ).filter( - constraint__source=group, # ...but exist for this group... - constraint__meeting=meeting, # ... at this meeting. - ).distinct() - - for inactive_constraint_name in inactive_cnames: - field_id = 'delete_{}'.format(inactive_constraint_name.slug) - self.fields[field_id] = forms.BooleanField(required=False, label='Delete this conflict', help_text='Delete this inactive conflict?') - constraints = group.constraint_source_set.filter(meeting=meeting, name=inactive_constraint_name) - self._inactive_wg_field_data.append( - (inactive_constraint_name, - ' '.join([c.target.acronym for c in constraints]), - field_id) - ) - - self.fields['joint_with_groups_selector'].widget.attrs['onchange'] = "document.form_post.joint_with_groups.value=document.form_post.joint_with_groups.value + ' ' + this.options[this.selectedIndex].value; return 1;" - self.fields["resources"].choices = [(x.pk,x.desc) for x in ResourceAssociation.objects.filter(name__used=True).order_by('name__order') ] - - if self.hidden: - # replace all the widgets to start... - for key in list(self.fields.keys()): - self.fields[key].widget = forms.HiddenInput() - # re-replace a couple special cases - self.fields['resources'].widget = forms.MultipleHiddenInput() - self.fields['timeranges'].widget = forms.MultipleHiddenInput() - # and entirely replace bethere - no need to support searching if input is hidden - self.fields['bethere'] = ModelMultipleChoiceField( - widget=forms.MultipleHiddenInput, required=False, - queryset=Person.objects.all(), - ) - - def wg_constraint_fields(self): - """Iterates over wg constraint fields - - Intended for use in the template. - """ - for cname, cfield_id, cselector_id in self._wg_field_data: - yield cname, self[cfield_id], self[cselector_id] - - def wg_constraint_count(self): - """How many wg constraints are there?""" - return len(self._wg_field_data) - - def wg_constraint_field_ids(self): - """Iterates over wg constraint field IDs""" - for cname, cfield_id, _ in self._wg_field_data: - yield cname, cfield_id - - def inactive_wg_constraints(self): - for cname, value, field_id in self._inactive_wg_field_data: - yield cname, value, self[field_id] - - def inactive_wg_constraint_count(self): - return len(self._inactive_wg_field_data) - - def inactive_wg_constraint_field_ids(self): - """Iterates over wg constraint field IDs""" - for cname, _, field_id in self._inactive_wg_field_data: - yield cname, field_id - - @staticmethod - def _add_widget_class(widget, new_class): - """Add a new class, taking care in case some already exist""" - existing_classes = widget.attrs.get('class', '').split() - widget.attrs['class'] = ' '.join(existing_classes + [new_class]) - - def _join_conflicts(self, cleaned_data, slugs): - """Concatenate constraint fields from cleaned data into a single list""" - conflicts = [] - for cname, cfield_id, _ in self._wg_field_data: - if cname.slug in slugs and cfield_id in cleaned_data: - groups = cleaned_data[cfield_id] - # convert to python list (allow space or comma separated lists) - items = groups.replace(',',' ').split() - conflicts.extend(items) - return conflicts - - def _validate_duplicate_conflicts(self, cleaned_data): - """Validate that no WGs appear in more than one constraint that does not allow duplicates - - Raises ValidationError - """ - # Only the older constraints (conflict, conflic2, conflic3) need to be mutually exclusive. - all_conflicts = self._join_conflicts(cleaned_data, ['conflict', 'conflic2', 'conflic3']) - seen = [] - duplicated = [] - errors = [] - for c in all_conflicts: - if c not in seen: - seen.append(c) - elif c not in duplicated: # only report once - duplicated.append(c) - errors.append(forms.ValidationError('%s appears in conflicts more than once' % c)) - return errors - - def clean_joint_with_groups(self): - groups = self.cleaned_data['joint_with_groups'] - check_conflict(groups, self.group) - return groups - - def clean_comments(self): - return clean_text_field(self.cleaned_data['comments']) - - def clean_bethere(self): - bethere = self.cleaned_data["bethere"] - if bethere: - extra = set( - Person.objects.filter( - role__group=self.group, role__name__in=["chair", "ad"] - ) - & bethere - ) - if extra: - extras = ", ".join(e.name for e in extra) - raise forms.ValidationError( - ( - f"Please remove the following person{pluralize(len(extra))}, the system " - f"tracks their availability due to their role{pluralize(len(extra))}: {extras}." - ) - ) - return bethere - - def clean_send_notifications(self): - return True if not self.notifications_optional else self.cleaned_data['send_notifications'] - - def is_valid(self): - return super().is_valid() and self.session_forms.is_valid() - - def clean(self): - super(SessionForm, self).clean() - self.session_forms.clean() - - data = self.cleaned_data - - # Validate the individual conflict fields - for _, cfield_id, _ in self._wg_field_data: - try: - check_conflict(data[cfield_id], self.group) - except forms.ValidationError as e: - self.add_error(cfield_id, e) - - # Skip remaining tests if individual field tests had errors, - if self.errors: - return data - - # error if conflicts contain disallowed dupes - for error in self._validate_duplicate_conflicts(data): - self.add_error(None, error) - - # Verify expected number of session entries are present - num_sessions_with_data = len(self.session_forms.forms_to_keep) - num_sessions_expected = -1 - try: - num_sessions_expected = int(data.get('num_session', '')) - except ValueError: - self.add_error('num_session', 'Invalid value for number of sessions') - if num_sessions_with_data < num_sessions_expected: - self.add_error('num_session', 'Must provide data for all sessions') - - # if default (empty) option is selected, cleaned_data won't include num_session key - if num_sessions_expected != 2 and num_sessions_expected is not None: - if data.get('session_time_relation'): - self.add_error( - 'session_time_relation', - forms.ValidationError('Time between sessions can only be used when two sessions are requested.') - ) - - joint_session = data.get('joint_for_session', '') - if joint_session != '': - joint_session = int(joint_session) - if joint_session > num_sessions_with_data: - self.add_error( - 'joint_for_session', - forms.ValidationError( - f'Session {joint_session} can not be the joint session, the session has not been requested.' - ) - ) - - return data - - @property - def media(self): - # get media for our formset - return super().media + self.session_forms.media + forms.Media(js=('secr/js/session_form.js',)) - - -# Used for totally virtual meetings during COVID-19 to omit the expected -# number of attendees since there were no room size limitations -# -# class VirtualSessionForm(SessionForm): -# '''A SessionForm customized for special virtual meeting requirements''' -# attendees = forms.IntegerField(required=False) - - -class ToolStatusForm(forms.Form): - message = forms.CharField(widget=forms.Textarea(attrs={'rows':'3','cols':'80'}), strip=False) - diff --git a/ietf/secr/sreq/urls.py b/ietf/secr/sreq/urls.py deleted file mode 100644 index 7e0db8117a..0000000000 --- a/ietf/secr/sreq/urls.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright The IETF Trust 2007-2019, All Rights Reserved - -from django.conf import settings - -from ietf.secr.sreq import views -from ietf.utils.urls import url - -urlpatterns = [ - url(r'^$', views.main), - url(r'^status/$', views.tool_status), - url(r'^%(acronym)s/$' % settings.URL_REGEXPS, views.view), - url(r'^(?P[A-Za-z0-9_\-\+]+)/%(acronym)s/view/$' % settings.URL_REGEXPS, views.view), - url(r'^%(acronym)s/approve/$' % settings.URL_REGEXPS, views.approve), - url(r'^%(acronym)s/cancel/$' % settings.URL_REGEXPS, views.cancel), - url(r'^%(acronym)s/confirm/$' % settings.URL_REGEXPS, views.confirm), - url(r'^%(acronym)s/edit/$' % settings.URL_REGEXPS, views.edit), - url(r'^%(acronym)s/new/$' % settings.URL_REGEXPS, views.new), - url(r'^%(acronym)s/no_session/$' % settings.URL_REGEXPS, views.no_session), - url(r'^(?P[A-Za-z0-9_\-\+]+)/%(acronym)s/edit/$' % settings.URL_REGEXPS, views.edit), -] diff --git a/ietf/secr/telechat/tests.py b/ietf/secr/telechat/tests.py index 39949b83a2..91ccde2187 100644 --- a/ietf/secr/telechat/tests.py +++ b/ietf/secr/telechat/tests.py @@ -13,6 +13,7 @@ IndividualDraftFactory, ConflictReviewFactory) from ietf.doc.models import BallotDocEvent, BallotType, BallotPositionDocEvent, State, Document from ietf.doc.utils import update_telechat, create_ballot_if_not_open +from ietf.meeting.factories import MeetingFactory from ietf.utils.test_utils import TestCase from ietf.utils.timezone import date_today, datetime_today from ietf.iesg.models import TelechatDate @@ -25,6 +26,26 @@ def augment_data(): TelechatDate.objects.create(date=date_today()) +class SecrUrlTests(TestCase): + def test_urls(self): + MeetingFactory(type_id='ietf', date=date_today()) + + # check public options + response = self.client.get("/secr/") + self.assertEqual(response.status_code, 200) + q = PyQuery(response.content) + links = q('div.secr-menu a') + self.assertEqual(len(links), 1) + self.assertEqual(PyQuery(links[0]).text(), 'Announcements') + + # check secretariat only options + self.client.login(username="secretary", password="secretary+password") + response = self.client.get("/secr/") + self.assertEqual(response.status_code, 200) + q = PyQuery(response.content) + links = q('div.secr-menu a') + self.assertEqual(len(links), 4) + class SecrTelechatTestCase(TestCase): def test_main(self): "Main Test" @@ -235,7 +256,7 @@ def test_doc_detail_post_update_state_action_holder_automation(self): self.assertEqual(response.status_code,302) draft = Document.objects.get(name=draft.name) self.assertEqual(draft.get_state('draft-iesg').slug,'defer') - self.assertCountEqual(draft.action_holders.all(), [draft.ad] + draft.authors()) + self.assertCountEqual(draft.action_holders.all(), [draft.ad] + draft.author_persons()) self.assertEqual(draft.docevent_set.filter(type='changed_action_holders').count(), 1) # Removing need-rev should remove authors @@ -252,7 +273,7 @@ def test_doc_detail_post_update_state_action_holder_automation(self): # Setting to approved should remove all action holders # noinspection DjangoOrm - draft.action_holders.add(*(draft.authors())) # add() with through model ok in Django 2.2+ + draft.action_holders.add(*(draft.author_persons())) # add() with through model ok in Django 2.2+ response = self.client.post(url,{ 'submit': 'update_state', 'state': State.objects.get(type_id='draft-iesg', slug='approved').pk, diff --git a/ietf/secr/templates/announcement/confirm.html b/ietf/secr/templates/announcement/confirm.html index ddf2a6de6e..0e1f72c54b 100644 --- a/ietf/secr/templates/announcement/confirm.html +++ b/ietf/secr/templates/announcement/confirm.html @@ -1,22 +1,17 @@ -{% extends "base_site.html" %} +{# Copyright The IETF Trust 2024, All Rights Reserved #} +{% extends "base.html" %} {% load static %} - +{% load ietf_filters %} +{% load django_bootstrap5 %} {% block title %}Announcement{% endblock %} - -{% block extrahead %}{{ block.super }} - -{% endblock %} - -{% block breadcrumbs %}{{ block.super }} - » Announcement -{% endblock %} - {% block content %} +

Announcement

+

Confirm Announcement

-
{% csrf_token %} + {% csrf_token %} 
 To: {{ to }}
@@ -29,15 +24,13 @@ 
Confirm Announcement

 {{ message.body }}
- {{ form }} -
-
    -
    • -
    • -
    • -
-
+ {% bootstrap_form form %} +
+ + + +
diff --git a/ietf/secr/templates/announcement/index.html b/ietf/secr/templates/announcement/index.html new file mode 100644 index 0000000000..ad7226e3bc --- /dev/null +++ b/ietf/secr/templates/announcement/index.html @@ -0,0 +1,31 @@ +{# Copyright The IETF Trust 2024, All Rights Reserved #} +{% extends "base.html" %} +{% load static %} +{% load ietf_filters %} +{% load django_bootstrap5 %} +{% block title %}Announcement{% endblock %} +{% block content %} +

Announcement

+ {% if form.non_field_errors %}
{{ form.non_field_errors }}
{% endif %} + +
+ {% csrf_token %} + {% bootstrap_field form.nomcom layout='horizontal' %} + {% bootstrap_field form.to layout='horizontal' %} + {% bootstrap_field form.to_custom layout='horizontal' %} + {% bootstrap_field form.frm layout='horizontal' %} + {% bootstrap_field form.cc layout='horizontal' %} + {% bootstrap_field form.bcc layout='horizontal' %} + {% bootstrap_field form.reply_to layout='horizontal' %} + {% bootstrap_field form.subject layout='horizontal' %} + {% bootstrap_field form.body layout='horizontal' %} + + + Cancel +
+ +{% endblock %} +{% block js %} + +{% endblock %} \ No newline at end of file diff --git a/ietf/secr/templates/announcement/main.html b/ietf/secr/templates/announcement/main.html deleted file mode 100644 index c88b4a2406..0000000000 --- a/ietf/secr/templates/announcement/main.html +++ /dev/null @@ -1,36 +0,0 @@ -{% extends "base_site.html" %} - -{% block title %}Announcement{% endblock %} - -{% block breadcrumbs %}{{ block.super }} - » Announcement -{% endblock %} - -{% block content %} - -
-

Announcement

- -
{% csrf_token %} - - - {% if form.non_field_errors %}{{ form.non_field_errors }}{% endif %} - {% for field in form.visible_fields %} - - - - - {% endfor %} - -
{{ field.label_tag }}{% if field.field.required %} *{% endif %}{{ field.errors }}{{ field }}{% if field.help_text %}
{{ field.help_text }}{% endif %}
-
-
    -
    • -
    • -
-
- -
-
- -{% endblock %} diff --git a/ietf/secr/templates/includes/activities.html b/ietf/secr/templates/includes/activities.html deleted file mode 100644 index 1304b7c48d..0000000000 --- a/ietf/secr/templates/includes/activities.html +++ /dev/null @@ -1,23 +0,0 @@ -

Activities Log

-
- - - - - - - - - - - {% for entry in activities %} - - - - - - - {% endfor %} - -
DateTimeActionName
{{ entry.act_date }}{{ entry.act_time }}{{ entry.activity }}{{ entry.act_by }}
-
diff --git a/ietf/secr/templates/includes/buttons_next_cancel.html b/ietf/secr/templates/includes/buttons_next_cancel.html deleted file mode 100644 index 95d25f55bc..0000000000 --- a/ietf/secr/templates/includes/buttons_next_cancel.html +++ /dev/null @@ -1,6 +0,0 @@ -
-
    -
    • -
    • -
-
diff --git a/ietf/secr/templates/includes/buttons_submit_cancel.html b/ietf/secr/templates/includes/buttons_submit_cancel.html deleted file mode 100644 index df40c98255..0000000000 --- a/ietf/secr/templates/includes/buttons_submit_cancel.html +++ /dev/null @@ -1,6 +0,0 @@ -
-
    -
    • -
    • -
-
diff --git a/ietf/secr/templates/includes/sessions_footer.html b/ietf/secr/templates/includes/sessions_footer.html deleted file mode 100755 index 2a26440047..0000000000 --- a/ietf/secr/templates/includes/sessions_footer.html +++ /dev/null @@ -1,5 +0,0 @@ - \ No newline at end of file diff --git a/ietf/secr/templates/includes/sessions_request_form.html b/ietf/secr/templates/includes/sessions_request_form.html deleted file mode 100755 index 61b1673357..0000000000 --- a/ietf/secr/templates/includes/sessions_request_form.html +++ /dev/null @@ -1,130 +0,0 @@ -* Required Field -
{% csrf_token %} - {{ form.session_forms.management_form }} - {% if form.non_field_errors %} - {{ form.non_field_errors }} - {% endif %} - - - - - - {% if group.features.acts_like_wg %} - - {% if not is_virtual %} - - {% endif %} - - {% else %}{# else not group.features.acts_like_wg #} - {% for session_form in form.session_forms %} - - {% endfor %} - {% endif %} - - - - - - - - - - {% if not is_virtual %} - - - - - - - - - - - - - - - - - - - {% endif %} - - - - - - {% if form.notifications_optional %} - - - - - {% endif %} - -
Working Group Name:{{ group.name }} ({{ group.acronym }})
Area Name:{% if group.parent %}{{ group.parent.name }} ({{ group.parent.acronym }}){% endif %}
Number of Sessions:*{{ form.num_session.errors }}{{ form.num_session }}
Session 1:*{% include 'meeting/session_details_form.html' with form=form.session_forms.0 hide_onsite_tool_prompt=True only %}
Session 2:*{% include 'meeting/session_details_form.html' with form=form.session_forms.1 hide_onsite_tool_prompt=True only %}
Time between two sessions:{{ form.session_time_relation.errors }}{{ form.session_time_relation }}
Additional Session Request:{{ form.third_session }} Check this box to request an additional session.
- Additional slot may be available after agenda scheduling has closed and with the approval of an Area Director.
-
- Third Session: - {% include 'meeting/session_details_form.html' with form=form.session_forms.2 hide_onsite_tool_prompt=True only %} -
-
Session {{ forloop.counter }}:*{% include 'meeting/session_details_form.html' with form=session_form only %}
Number of Attendees:{% if not is_virtual %}*{% endif %}{{ form.attendees.errors }}{{ form.attendees }}
Participants who must be present: - {{ form.bethere.errors }} - {{ form.bethere }} -

- Do not include Area Directors and WG Chairs; the system already tracks their availability. -

-
Conflicts to Avoid: - - - - - - - {% for cname, cfield, cselector in form.wg_constraint_fields %} - - {% if forloop.first %}{% endif %} - - - - {% empty %}{# shown if there are no constraint fields #} - - {% endfor %} - {% if form.inactive_wg_constraints %} - {% for cname, value, field in form.inactive_wg_constraints %} - - {% if forloop.first %} - - {% endif %} - - - - {% endfor %} - {% endif %} - - - - - -
Other WGs that included {{ group.name }} in their conflict lists:{{ session_conflicts.inbound|default:"None" }}
WG Sessions:
You may select multiple WGs within each category		{{ cname|title }}{{ cselector }} -
- {{ cfield.errors }}{{ cfield }} -
No constraints are enabled for this meeting.
- Disabled for this meeting - {{ cname|title }}
{{ field }} {{ field.label }}
BOF Sessions:If the sessions can not be found in the fields above, please enter free form requests in the Special Requests field below.
-
Resources requested: - {{ form.resources.errors }} {{ form.resources }} -
Times during which this WG can not meet:
Please explain any selections in Special Requests below.		{{ form.timeranges.errors }}{{ form.timeranges }}
- Plan session adjacent with another WG:
- (Immediately before or after another WG, no break in between, in the same room.) - 		{{ form.adjacent_with_wg.errors }}{{ form.adjacent_with_wg }}
- Joint session with:
- (To request one session for multiple WGs together.) - 		To request a joint session with another group, please contact the secretariat.
Special Requests:
 
i.e. restrictions on meeting times / days, etc.
(limit 200 characters)		{{ form.comments.errors }}{{ form.comments }}
{{ form.send_notifications.label }}{{ form.send_notifications.errors }}{{ form.send_notifications }}
- -
-
    -
    • -
    • -
-
-
\ No newline at end of file diff --git a/ietf/secr/templates/includes/sessions_request_view.html b/ietf/secr/templates/includes/sessions_request_view.html deleted file mode 100644 index bc6aef0611..0000000000 --- a/ietf/secr/templates/includes/sessions_request_view.html +++ /dev/null @@ -1,73 +0,0 @@ -{% load ams_filters %} - - - - - - {% if form %} - {% include 'includes/sessions_request_view_formset.html' with formset=form.session_forms group=group session=session only %} - {% else %} - {% include 'includes/sessions_request_view_session_set.html' with session_set=sessions group=group session=session only %} - {% endif %} - - - - - - - - - - {% if not is_virtual %} - - - - - {% endif %} - - - - - - - - - {% if not is_virtual %} - - - - - - - - - {% endif %} - - {% if form and form.notifications_optional %} - - - - - {% endif %} - -
Working Group Name:{{ group.name }} ({{ group.acronym }})
Area Name:{{ group.parent }}
Number of Sessions Requested:{% if session.third_session %}3{% else %}{{ session.num_session }}{% endif %}
Number of Attendees:{{ session.attendees }}
Conflicts to Avoid: - {% if session_conflicts.outbound %} - - - {% for conflict in session_conflicts.outbound %} - - {% endfor %} - -
{{ conflict.name|title }}: {{ conflict.groups }}
- {% else %}None{% endif %} -
Other WGs that included {{ group }} in their conflict list:{% if session_conflicts.inbound %}{{ session_conflicts.inbound }}{% else %}None so far{% endif %}
Resources requested:{% if session.resources %}
    {% for resource in session.resources %}
  • {{ resource.desc }}
    • {% endfor %}
{% else %}None so far{% endif %}
Participants who must be present:{% if session.bethere %}
    {% for person in session.bethere %}
  • {{ person }}
    • {% endfor %}
{% else %}None{% endif %}
Can not meet on:{% if session.timeranges_display %}{{ session.timeranges_display|join:', ' }}{% else %}No constraints{% endif %}
Adjacent with WG:{{ session.adjacent_with_wg|default:'No preference' }}
Joint session: - {% if session.joint_with_groups %} - {{ session.joint_for_session_display }} with: {{ session.joint_with_groups }} - {% else %} - Not a joint session - {% endif %} -
Special Requests:{{ session.comments }}
- {{ form.send_notifications.label}} - - {% if form.cleaned_data.send_notifications %}Yes{% else %}No{% endif %} -
\ No newline at end of file diff --git a/ietf/secr/templates/includes/sessions_request_view_formset.html b/ietf/secr/templates/includes/sessions_request_view_formset.html deleted file mode 100644 index 80cad8d829..0000000000 --- a/ietf/secr/templates/includes/sessions_request_view_formset.html +++ /dev/null @@ -1,32 +0,0 @@ -{% load ams_filters %}{# keep this in sync with sessions_request_view_session_set.html #} -{% for sess_form in formset %}{% if sess_form.cleaned_data and not sess_form.cleaned_data.DELETE %} - - Session {{ forloop.counter }}: - -
-
Length
-
{{ sess_form.cleaned_data.requested_duration.total_seconds|display_duration }}
- {% if sess_form.cleaned_data.name %} -
Name
-
{{ sess_form.cleaned_data.name }}
{% endif %} - {% if sess_form.cleaned_data.purpose.slug != 'regular' %} -
Purpose
-
- {{ sess_form.cleaned_data.purpose }} - {% if sess_form.cleaned_data.purpose.timeslot_types|length > 1 %}({{ sess_form.cleaned_data.type }} - ){% endif %} -
-
Onsite tool?
-
{{ sess_form.cleaned_data.has_onsite_tool|yesno }}
- {% endif %} -
- - - {% if group.features.acts_like_wg and forloop.counter == 2 and not is_virtual %} - - Time between sessions: - {% if session.session_time_relation_display %}{{ session.session_time_relation_display }}{% else %}No - preference{% endif %} - - {% endif %} -{% endif %}{% endfor %} \ No newline at end of file diff --git a/ietf/secr/templates/includes/sessions_request_view_session_set.html b/ietf/secr/templates/includes/sessions_request_view_session_set.html deleted file mode 100644 index a434b9d22b..0000000000 --- a/ietf/secr/templates/includes/sessions_request_view_session_set.html +++ /dev/null @@ -1,32 +0,0 @@ -{% load ams_filters %}{# keep this in sync with sessions_request_view_formset.html #} -{% for sess in session_set %} - - Session {{ forloop.counter }}: - -
-
Length
-
{{ sess.requested_duration.total_seconds|display_duration }}
- {% if sess.name %} -
Name
-
{{ sess.name }}
{% endif %} - {% if sess.purpose.slug != 'regular' %} -
Purpose
-
- {{ sess.purpose }} - {% if sess.purpose.timeslot_types|length > 1 %}({{ sess.type }} - ){% endif %} -
-
Onsite tool?
-
{{ sess.has_onsite_tool|yesno }}
- {% endif %} -
- - - {% if group.features.acts_like_wg and forloop.counter == 2 and not is_virtual %} - - Time between sessions: - {% if session.session_time_relation_display %}{{ session.session_time_relation_display }}{% else %}No - preference{% endif %} - - {% endif %} -{% endfor %} \ No newline at end of file diff --git a/ietf/secr/templates/index.html b/ietf/secr/templates/index.html new file mode 100644 index 0000000000..9ea7021279 --- /dev/null +++ b/ietf/secr/templates/index.html @@ -0,0 +1,31 @@ +{# Copyright The IETF Trust 2007-2025, All Rights Reserved #} +{% extends "base.html" %} +{% load static %} +{% load ietf_filters %} +{% block title %}Secretariat Dashboard{% endblock %} +{% block content %} +

Secretariat Dashboard

+
+ {% if user|has_role:"Secretariat" %} +

IESG

+ + +

IDs and WGs Process

+ + +

Meetings and Proceedings

+ + {% else %} + + {% endif %} +
+{% endblock %} \ No newline at end of file diff --git a/ietf/secr/templates/main.html b/ietf/secr/templates/main.html deleted file mode 100644 index 42d6e8f6a1..0000000000 --- a/ietf/secr/templates/main.html +++ /dev/null @@ -1,69 +0,0 @@ -{% extends "base_site.html" %} -{% load ietf_filters %} - -{% block content %} -
- - {% if user|has_role:"Secretariat" %} - - - - - - - - - - - - -
-

IESG

- - 		-

IDs and WGs Process

- -
-

Meetings and Proceedings

- -
- - {% else %} - - - - - - - - - - - - -
-

Section 1

- - 		-

Section 2

-
-

Section 3

- 		-

Section 4

-
- - {% endif %} - -
-{% endblock %} \ No newline at end of file diff --git a/ietf/secr/templates/sreq/confirm.html b/ietf/secr/templates/sreq/confirm.html deleted file mode 100755 index 025375af32..0000000000 --- a/ietf/secr/templates/sreq/confirm.html +++ /dev/null @@ -1,57 +0,0 @@ -{% extends "base_site.html" %} -{% load static %} - -{% block title %}Sessions - Confirm{% endblock %} - -{% block extrastyle %} - -{% endblock %} - -{% block extrahead %}{{ block.super }} - - {{ form.media }} -{% endblock %} - -{% block breadcrumbs %}{{ block.super }} - » Sessions - » New - » Session Request Confirmation -{% endblock %} - -{% block content %} - -
-

Sessions - Confirm

- - {% include "includes/sessions_request_view.html" %} - - {% if group.features.acts_like_wg and form.session_forms.forms_to_keep|length > 2 %} -
-

- - Note: Your request for a third session must be approved by an area director before - being submitted to agenda@ietf.org. Click "Submit" below to email an approval - request to the area directors. - -

-
- {% endif %} - -
- {% csrf_token %} - {{ form }} - {{ form.session_forms.management_form }} - {% for sf in form.session_forms %} - {% include 'meeting/session_details_form.html' with form=sf hidden=True only %} - {% endfor %} - {% include "includes/buttons_submit_cancel.html" %} -
- -
- -{% endblock %} \ No newline at end of file diff --git a/ietf/secr/templates/sreq/edit.html b/ietf/secr/templates/sreq/edit.html deleted file mode 100755 index f6e62104b0..0000000000 --- a/ietf/secr/templates/sreq/edit.html +++ /dev/null @@ -1,39 +0,0 @@ -{% extends "base_site.html" %} -{% load static %} -{% block title %}Sessions - Edit{% endblock %} - -{% block extrahead %}{{ block.super }} - - - {{ form.media }} - -{% endblock %} - -{% block breadcrumbs %}{{ block.super }} - » Sessions - » {{ group.acronym }} - » Edit -{% endblock %} - -{% block instructions %} - Instructions -{% endblock %} - -{% block content %} -
-

IETF {{ meeting.number }}: Edit Session Request

-
-
- - {% include "includes/sessions_request_form.html" %} - -
-
-{% endblock %} - -{% block footer-extras %} - {% include "includes/sessions_footer.html" %} -{% endblock %} \ No newline at end of file diff --git a/ietf/secr/templates/sreq/locked.html b/ietf/secr/templates/sreq/locked.html deleted file mode 100755 index c27cf578ed..0000000000 --- a/ietf/secr/templates/sreq/locked.html +++ /dev/null @@ -1,30 +0,0 @@ -{% extends "base_site.html" %} -{% load static %} - -{% block title %}Sessions{% endblock %} - -{% block extrahead %}{{ block.super }} - -{% endblock %} - -{% block breadcrumbs %}{{ block.super }} - » Sessions (Locked) -{% endblock %} - -{% block content %} -

» View list of timeslot requests

-
-

Sessions - Status

- -

{{ message }}

- -
-
    -
    • -
-
- - -
- -{% endblock %} \ No newline at end of file diff --git a/ietf/secr/templates/sreq/main.html b/ietf/secr/templates/sreq/main.html deleted file mode 100755 index a6695cd4f3..0000000000 --- a/ietf/secr/templates/sreq/main.html +++ /dev/null @@ -1,65 +0,0 @@ -{% extends "base_site.html" %} -{% load ietf_filters %} -{% load static %} - -{% block title %}Sessions{% endblock %} - -{% block extrahead %}{{ block.super }} - -{% endblock %} - -{% block breadcrumbs %}{{ block.super }} - » Sessions -{% endblock %} -{% block instructions %} - Instructions -{% endblock %} - -{% block content %} -

» View list of timeslot requests

-
-

- Sessions Request Tool: IETF {{ meeting.number }} - {% if user|has_role:"Secretariat" %} - {% if is_locked %} - Tool Status: Locked - {% else %} - Tool Status: Unlocked - {% endif %} - {% endif %} -

-
-

Request New Session

-

The list below includes those working groups that you currently chair which do not already have a session scheduled. You can click on an acronym to initiate a request for a new session at the upcoming IETF meeting. Click "Group will not meet" to send a notification that the group does not plan to meet.

-
    - {% for group in unscheduled_groups %} -
  • - {{ group.acronym }} - {% if group.not_meeting %} - (Currently, this group does not plan to hold a session at IETF {{ meeting.number }}) - {% endif %} -
    • - {% empty %} -
  • NONE
    • - {% endfor %} -
-
-
-
-

Edit / Cancel Previously Requested Sessions

-

The list below includes those working groups for which you or your co-chair has requested sessions at the upcoming IETF meeting. You can click on an acronym to initiate changes to a session, or cancel a session.

- -
-
- -{% endblock %} - -{% block footer-extras %} - {% include "includes/sessions_footer.html" %} -{% endblock %} \ No newline at end of file diff --git a/ietf/secr/templates/sreq/new.html b/ietf/secr/templates/sreq/new.html deleted file mode 100755 index 3f46e6f897..0000000000 --- a/ietf/secr/templates/sreq/new.html +++ /dev/null @@ -1,43 +0,0 @@ -{% extends "base_site.html" %} -{% load static %} - -{% block title %}Sessions- New{% endblock %} - -{% block extrahead %}{{ block.super }} - - - {{ form.media }} - -{% endblock %} - -{% block breadcrumbs %}{{ block.super }} - » Sessions - » New Session Request -{% endblock %} - -{% block instructions %} - Instructions -{% endblock %} - -{% block content %} -
-

IETF {{ meeting.number }}: New Session Request

-
-
-
    -
    • -
    • -
-
- {% include "includes/sessions_request_form.html" %} - -
- -{% endblock %} - -{% block footer-extras %} - {% include "includes/sessions_footer.html" %} -{% endblock %} \ No newline at end of file diff --git a/ietf/secr/templates/sreq/tool_status.html b/ietf/secr/templates/sreq/tool_status.html deleted file mode 100755 index b91e73a129..0000000000 --- a/ietf/secr/templates/sreq/tool_status.html +++ /dev/null @@ -1,42 +0,0 @@ -{% extends "base_site.html" %} -{% load static %} - -{% block title %}Sessions{% endblock %} - -{% block extrahead %}{{ block.super }} - -{% endblock %} - -{% block breadcrumbs %}{{ block.super }} - » Sessions - » Session Status -{% endblock %} - -{% block content %} - -
-

Sessions - Status

-

Enter the message that you would like displayed to the WG Chair when this tool is locked.

-
{% csrf_token %} - - - - {{ form.as_table }} - -
-
-
    - {% if is_locked %} -
    • - {% else %} -
    • - {% endif %} -
    • -
-
- -
- -
- -{% endblock %} diff --git a/ietf/secr/templates/sreq/view.html b/ietf/secr/templates/sreq/view.html deleted file mode 100644 index 9a0a3b01c1..0000000000 --- a/ietf/secr/templates/sreq/view.html +++ /dev/null @@ -1,55 +0,0 @@ -{% extends "base_site.html" %} -{% load static %} - -{% block title %}Sessions - View{% endblock %} - -{% block extrahead %}{{ block.super }} - -{% endblock %} - -{% block extrastyle %} - -{% endblock %} - -{% block breadcrumbs %}{{ block.super }} - » Sessions - » {{ group.acronym }} -{% endblock %} - -{% block instructions %} - Instructions -{% endblock %} - -{% block content %} - -
-

Sessions - View (meeting: {{ meeting.number }})

- - {% include "includes/sessions_request_view.html" %} - -
- - {% include "includes/activities.html" %} - -
-
    -
    • - {% if show_approve_button %} -
    • - {% endif %} -
    • -
    • -
-
-
- -{% endblock %} - -{% block footer-extras %} - {% include "includes/sessions_footer.html" %} -{% endblock %} diff --git a/ietf/secr/urls.py b/ietf/secr/urls.py index 0ce14a449a..ab21046654 100644 --- a/ietf/secr/urls.py +++ b/ietf/secr/urls.py @@ -1,11 +1,22 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.conf import settings from django.urls import re_path, include from django.views.generic import TemplateView +from django.views.generic.base import RedirectView urlpatterns = [ - re_path(r'^$', TemplateView.as_view(template_name='main.html')), + re_path(r'^$', TemplateView.as_view(template_name='index.html'), name='ietf.secr'), re_path(r'^announcement/', include('ietf.secr.announcement.urls')), re_path(r'^meetings/', include('ietf.secr.meetings.urls')), re_path(r'^rolodex/', include('ietf.secr.rolodex.urls')), - re_path(r'^sreq/', include('ietf.secr.sreq.urls')), + # remove these redirects after 125 + re_path(r'^sreq/$', RedirectView.as_view(url='/meeting/session/request/', permanent=True)), + re_path(r'^sreq/%(acronym)s/$' % settings.URL_REGEXPS, RedirectView.as_view(url='/meeting/session/request/%(acronym)s/view/', permanent=True)), + re_path(r'^sreq/%(acronym)s/edit/$' % settings.URL_REGEXPS, RedirectView.as_view(url='/meeting/session/request/%(acronym)s/edit/', permanent=True)), + re_path(r'^sreq/%(acronym)s/new/$' % settings.URL_REGEXPS, RedirectView.as_view(url='/meeting/session/request/%(acronym)s/new/', permanent=True)), + re_path(r'^sreq/(?P[A-Za-z0-9_\-\+]+)/%(acronym)s/view/$' % settings.URL_REGEXPS, RedirectView.as_view(url='/meeting/%(num)s/session/request/%(acronym)s/view/', permanent=True)), + re_path(r'^sreq/(?P[A-Za-z0-9_\-\+]+)/%(acronym)s/edit/$' % settings.URL_REGEXPS, RedirectView.as_view(url='/meeting/%(num)s/session/request/%(acronym)s/edit/', permanent=True)), + # --------------------------------- re_path(r'^telechat/', include('ietf.secr.telechat.urls')), ] diff --git a/ietf/secr/utils/group.py b/ietf/secr/utils/group.py deleted file mode 100644 index 40a9065ace..0000000000 --- a/ietf/secr/utils/group.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright The IETF Trust 2013-2020, All Rights Reserved -# -*- coding: utf-8 -*- - - -# Python imports - -# Django imports -from django.core.exceptions import ObjectDoesNotExist - -# Datatracker imports -from ietf.group.models import Group -from ietf.ietfauth.utils import has_role - - -def get_my_groups(user,conclude=False): - ''' - Takes a Django user object (from request) - Returns a list of groups the user has access to. Rules are as follows - secretariat - has access to all groups - area director - has access to all groups in their area - wg chair or secretary - has access to their own group - chair of irtf has access to all irtf groups - - If user=None than all groups are returned. - concluded=True means include concluded groups. Need this to upload materials for groups - after they've been concluded. it happens. - ''' - my_groups = set() - states = ['bof','proposed','active'] - if conclude: - states.extend(['conclude','bof-conc']) - - all_groups = Group.objects.filter(type__features__has_meetings=True, state__in=states).order_by('acronym') - if user == None or has_role(user,'Secretariat'): - return all_groups - - try: - person = user.person - except ObjectDoesNotExist: - return list() - - for group in all_groups: - if group.role_set.filter(person=person,name__in=('chair','secr','ad')): - my_groups.add(group) - continue - if group.parent and group.parent.role_set.filter(person=person,name__in=('ad','chair')): - my_groups.add(group) - continue - - return list(my_groups) diff --git a/ietf/settings.py b/ietf/settings.py index 33a2f976d9..95f2ffefd7 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2007-2024, All Rights Reserved +# Copyright The IETF Trust 2007-2026, All Rights Reserved # -*- coding: utf-8 -*- @@ -9,26 +9,42 @@ import os import sys import datetime +import pathlib import warnings from hashlib import sha384 from typing import Any, Dict, List, Tuple # pyflakes:ignore +from django.http import UnreadablePostError +# DeprecationWarnings are suppressed by default, enable them warnings.simplefilter("always", DeprecationWarning) -warnings.filterwarnings("ignore", message="pkg_resources is deprecated as an API") -warnings.filterwarnings("ignore", "Log out via GET requests is deprecated") # happens in oidc_provider -warnings.filterwarnings("ignore", module="tastypie", message="The django.utils.datetime_safe module is deprecated.") -warnings.filterwarnings("ignore", module="oidc_provider", message="The django.utils.timezone.utc alias is deprecated.") + +# Warnings that must be resolved for Django 5.x +warnings.filterwarnings("ignore", "Log out via GET requests is deprecated") # caused by oidc_provider +warnings.filterwarnings("ignore", message="The django.utils.timezone.utc alias is deprecated.", module="oidc_provider") +warnings.filterwarnings("ignore", message="The django.utils.datetime_safe module is deprecated.", module="tastypie") warnings.filterwarnings("ignore", message="The USE_DEPRECATED_PYTZ setting,") # https://github.com/ietf-tools/datatracker/issues/5635 +warnings.filterwarnings("ignore", message="The is_dst argument to make_aware\\(\\)") # caused by django-filters when USE_DEPRECATED_PYTZ is true warnings.filterwarnings("ignore", message="The USE_L10N setting is deprecated.") # https://github.com/ietf-tools/datatracker/issues/5648 warnings.filterwarnings("ignore", message="django.contrib.auth.hashers.CryptPasswordHasher is deprecated.") # https://github.com/ietf-tools/datatracker/issues/5663 -warnings.filterwarnings("ignore", message="'urllib3\\[secure\\]' extra is deprecated") -warnings.filterwarnings("ignore", message="The logout\\(\\) view is superseded by") + +# Other DeprecationWarnings +warnings.filterwarnings("ignore", message="pkg_resources is deprecated as an API", module="pyang.plugin") warnings.filterwarnings("ignore", message="Report.file_reporters will no longer be available in Coverage.py 4.2", module="coverage.report") -warnings.filterwarnings("ignore", message="Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated", module="bleach") -warnings.filterwarnings("ignore", message="HTTPResponse.getheader\\(\\) is deprecated", module='selenium.webdriver') +warnings.filterwarnings("ignore", message="currentThread\\(\\) is deprecated", module="coverage.pytracer") +warnings.filterwarnings("ignore", message="co_lnotab is deprecated", module="coverage.parser") +warnings.filterwarnings("ignore", message="datetime.datetime.utcnow\\(\\) is deprecated", module="botocore.auth") +warnings.filterwarnings("ignore", message="datetime.datetime.utcnow\\(\\) is deprecated", module="oic.utils.time_util") +warnings.filterwarnings("ignore", message="datetime.datetime.utcfromtimestamp\\(\\) is deprecated", module="oic.utils.time_util") +warnings.filterwarnings("ignore", message="datetime.datetime.utcfromtimestamp\\(\\) is deprecated", module="pytz.tzinfo") +warnings.filterwarnings("ignore", message="'instantiateVariableFont' is deprecated", module="weasyprint") + -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(os.path.abspath(BASE_DIR + "/..")) +base_path = pathlib.Path(__file__).resolve().parent +BASE_DIR = str(base_path) + +project_path = base_path.parent +PROJECT_DIR = str(project_path) +sys.path.append(PROJECT_DIR) from ietf import __version__ import debug @@ -61,6 +77,26 @@ 'django.contrib.auth.hashers.CryptPasswordHasher', ] + +PASSWORD_POLICY_MIN_LENGTH = 12 +PASSWORD_POLICY_ENFORCE_AT_LOGIN = False # should turn this on for prod + +AUTH_PASSWORD_VALIDATORS = [ + { + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", + "OPTIONS": { + "min_length": PASSWORD_POLICY_MIN_LENGTH, + } + }, + { + "NAME": "ietf.ietfauth.password_validation.StrongPasswordValidator", + }, +] +# In dev environments, settings_local overrides the password validators. Save +# a handle to the original value so settings_test can restore it so tests match +# production. +ORIG_AUTH_PASSWORD_VALIDATORS = AUTH_PASSWORD_VALIDATORS + ALLOWED_HOSTS = [".ietf.org", ".ietf.org.", "209.208.19.216", "4.31.198.44", "127.0.0.1", "localhost", ] # Server name of the tools server @@ -185,163 +221,131 @@ ENABLE_BLOBSTORAGE = True -BLOBSTORAGE_MAX_ATTEMPTS = 1 -BLOBSTORAGE_CONNECT_TIMEOUT = 2 -BLOBSTORAGE_READ_TIMEOUT = 2 +# "standard" retry mode is used, which does exponential backoff with a base factor of 2 +# and a cap of 20. +BLOBSTORAGE_MAX_ATTEMPTS = 5 # boto3 default is 3 (for "standard" retry mode) +BLOBSTORAGE_CONNECT_TIMEOUT = 10 # seconds; boto3 default is 60 +BLOBSTORAGE_READ_TIMEOUT = 10 # seconds; boto3 default is 60 + +# Caching for agenda data in seconds +AGENDA_CACHE_TIMEOUT_DEFAULT = 8 * 24 * 60 * 60 # 8 days +AGENDA_CACHE_TIMEOUT_CURRENT_MEETING = 6 * 60 # 6 minutes + WSGI_APPLICATION = "ietf.wsgi.application" AUTHENTICATION_BACKENDS = ( 'ietf.ietfauth.backends.CaseInsensitiveModelBackend', ) -FILE_UPLOAD_PERMISSIONS = 0o644 +FILE_UPLOAD_PERMISSIONS = 0o644 -# ------------------------------------------------------------------------ -# Django/Python Logging Framework Modifications +FIRST_V3_RFC = 8650 -# Filter out "Invalid HTTP_HOST" emails -# Based on http://www.tiwoc.de/blog/2013/03/django-prevent-email-notification-on-suspiciousoperation/ -from django.core.exceptions import SuspiciousOperation -def skip_suspicious_operations(record): - if record.exc_info: - exc_value = record.exc_info[1] - if isinstance(exc_value, SuspiciousOperation): - return False - return True -# Filter out UreadablePostError: -from django.http import UnreadablePostError +# +# Logging config +# + +# Callback to filter out UnreadablePostError: def skip_unreadable_post(record): if record.exc_info: - exc_type, exc_value = record.exc_info[:2] # pylint: disable=unused-variable + exc_type, exc_value = record.exc_info[:2] # pylint: disable=unused-variable if isinstance(exc_value, UnreadablePostError): return False return True -# Copied from DEFAULT_LOGGING as of Django 1.10.5 on 22 Feb 2017, and modified -# to incorporate html logging, invalid http_host filtering, and more. -# Changes from the default has comments. - -# The Python logging flow is as follows: -# (see https://docs.python.org/2.7/howto/logging.html#logging-flow) -# -# Init: get a Logger: logger = logging.getLogger(name) -# -# Logging call, e.g. logger.error(level, msg, *args, exc_info=(...), extra={...}) -# --> Logger (discard if level too low for this logger) -# (create log record from level, msg, args, exc_info, extra) -# --> Filters (discard if any filter attach to logger rejects record) -# --> Handlers (discard if level too low for handler) -# --> Filters (discard if any filter attached to handler rejects record) -# --> Formatter (format log record and emit) -# - LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - # - 'loggers': { - 'django': { - 'handlers': ['console', 'mail_admins'], - 'level': 'INFO', - }, - 'django.request': { - 'handlers': ['console'], - 'level': 'ERROR', + "version": 1, + "disable_existing_loggers": False, + "loggers": { + "celery": { + "handlers": ["console"], + "level": "INFO", }, - 'django.server': { - 'handlers': ['django.server'], - 'level': 'INFO', + "datatracker": { + "handlers": ["console"], + "level": "INFO", }, - 'django.security': { - 'handlers': ['console', ], - 'level': 'INFO', + "django": { + "handlers": ["console", "mail_admins"], + "level": "INFO", }, - 'oidc_provider': { - 'handlers': ['console', ], - 'level': 'DEBUG', + "django.request": {"level": "ERROR"}, # only log 5xx, ignore 4xx + "django.security": { + # SuspiciousOperation errors - log to console only + "handlers": ["console"], + "propagate": False, # no further handling please }, - 'datatracker': { - 'handlers': ['console'], - 'level': 'INFO', + "django.server": { + # Only used by Django's runserver development server + "handlers": ["django.server"], + "level": "INFO", }, - 'celery': { - 'handlers': ['console'], - 'level': 'INFO', + "oidc_provider": { + "handlers": ["console"], + "level": "DEBUG", }, }, - # - # No logger filters - # - 'handlers': { - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - 'formatter': 'plain', + "handlers": { + "console": { + "level": "DEBUG", + "class": "logging.StreamHandler", + "formatter": "plain", }, - 'debug_console': { - # Active only when DEBUG=True - 'level': 'DEBUG', - 'filters': ['require_debug_true'], - 'class': 'logging.StreamHandler', - 'formatter': 'plain', + "debug_console": { + "level": "DEBUG", + "filters": ["require_debug_true"], + "class": "logging.StreamHandler", + "formatter": "plain", }, - 'django.server': { - 'level': 'INFO', - 'class': 'logging.StreamHandler', - 'formatter': 'django.server', + "django.server": { + "level": "INFO", + "class": "logging.StreamHandler", + "formatter": "django.server", }, - 'mail_admins': { - 'level': 'ERROR', - 'filters': [ - 'require_debug_false', - 'skip_suspicious_operations', # custom - 'skip_unreadable_posts', # custom + "mail_admins": { + "level": "ERROR", + "filters": [ + "require_debug_false", + "skip_unreadable_posts", ], - 'class': 'django.utils.log.AdminEmailHandler', - 'include_html': True, # non-default - } + "class": "django.utils.log.AdminEmailHandler", + "include_html": True, + }, }, - # # All these are used by handlers - 'filters': { - 'require_debug_false': { - '()': 'django.utils.log.RequireDebugFalse', + "filters": { + "require_debug_false": { + "()": "django.utils.log.RequireDebugFalse", }, - 'require_debug_true': { - '()': 'django.utils.log.RequireDebugTrue', - }, - # custom filter, function defined above: - 'skip_suspicious_operations': { - '()': 'django.utils.log.CallbackFilter', - 'callback': skip_suspicious_operations, + "require_debug_true": { + "()": "django.utils.log.RequireDebugTrue", }, # custom filter, function defined above: - 'skip_unreadable_posts': { - '()': 'django.utils.log.CallbackFilter', - 'callback': skip_unreadable_post, + "skip_unreadable_posts": { + "()": "django.utils.log.CallbackFilter", + "callback": skip_unreadable_post, }, }, - # And finally the formatters - 'formatters': { - 'django.server': { - '()': 'django.utils.log.ServerFormatter', - 'format': '[%(server_time)s] %(message)s', + "formatters": { + "django.server": { + "()": "django.utils.log.ServerFormatter", + "format": "[%(server_time)s] %(message)s", }, - 'plain': { - 'style': '{', - 'format': '{levelname}: {name}:{lineno}: {message}', + "plain": { + "style": "{", + "format": "{levelname}: {name}:{lineno}: {message}", }, - 'json' : { + "json": { "class": "ietf.utils.jsonlogger.DatatrackerJsonFormatter", "style": "{", - "format": "{asctime}{levelname}{message}{name}{pathname}{lineno}{funcName}{process}", - } + "format": ( + "{asctime}{levelname}{message}{name}{pathname}{lineno}{funcName}" + "{process}{status_code}" + ), + }, }, } -# End logging -# ------------------------------------------------------------------------ - X_FRAME_OPTIONS = 'SAMEORIGIN' CSRF_TRUSTED_ORIGINS = [ @@ -375,6 +379,7 @@ def skip_unreadable_post(record): ], 'OPTIONS': { 'context_processors': [ + 'ietf.context_processors.traceparent_id', 'django.contrib.auth.context_processors.auth', 'django.template.context_processors.debug', # makes 'sql_queries' available in templates 'django.template.context_processors.i18n', @@ -407,12 +412,14 @@ def skip_unreadable_post(record): MIDDLEWARE = [ + "ietf.middleware.add_otel_traceparent_header", "django.middleware.csrf.CsrfViewMiddleware", "corsheaders.middleware.CorsMiddleware", # see docs on CORS_REPLACE_HTTPS_REFERER before using it "django.middleware.common.CommonMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", + "ietf.middleware.is_authenticated_header_middleware", "django.middleware.http.ConditionalGetMiddleware", "simple_history.middleware.HistoryRequestMiddleware", # comment in this to get logging of SQL insert and update statements: @@ -420,23 +427,23 @@ def skip_unreadable_post(record): "ietf.middleware.SMTPExceptionMiddleware", "ietf.middleware.Utf8ExceptionMiddleware", "ietf.middleware.redirect_trailing_period_middleware", - "django_referrer_policy.middleware.ReferrerPolicyMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", "django.middleware.security.SecurityMiddleware", - #"csp.middleware.CSPMiddleware", "ietf.middleware.unicode_nfkc_normalization_middleware", - "ietf.middleware.is_authenticated_header_middleware", ] ROOT_URLCONF = 'ietf.urls' -DJANGO_VITE_ASSETS_PATH = os.path.join(BASE_DIR, 'static/dist-neue') +# Configure django_vite +DJANGO_VITE: dict = {"default": {}} if DEBUG: - DJANGO_VITE_MANIFEST_PATH = os.path.join(BASE_DIR, 'static/dist-neue/manifest.json') + DJANGO_VITE["default"]["manifest_path"] = os.path.join( + BASE_DIR, 'static/dist-neue/manifest.json' + ) # Additional locations of static files (in addition to each app's static/ dir) STATICFILES_DIRS = ( - DJANGO_VITE_ASSETS_PATH, + os.path.join(BASE_DIR, "static/dist-neue"), # for django_vite os.path.join(BASE_DIR, 'static/dist'), os.path.join(BASE_DIR, 'secr/static/dist'), ) @@ -461,6 +468,7 @@ def skip_unreadable_post(record): 'django_celery_results', 'corsheaders', 'django_markup', + 'django_filters', 'oidc_provider', 'drf_spectacular', 'drf_standardized_errors', @@ -471,6 +479,7 @@ def skip_unreadable_post(record): 'widget_tweaks', # IETF apps 'ietf.api', + 'ietf.blobdb', 'ietf.community', 'ietf.dbtemplate', 'ietf.doc', @@ -499,7 +508,6 @@ def skip_unreadable_post(record): 'ietf.secr.announcement', 'ietf.secr.meetings', 'ietf.secr.rolodex', - 'ietf.secr.sreq', 'ietf.secr.telechat', ] @@ -539,8 +547,6 @@ def skip_unreadable_post(record): CORS_ALLOW_METHODS = ( 'GET', 'OPTIONS', ) CORS_URLS_REGEX = r'^(/api/.*|.*\.json|.*/json/?)$' -# Setting for django_referrer_policy.middleware.ReferrerPolicyMiddleware -REFERRER_POLICY = 'strict-origin-when-cross-origin' # django.middleware.security.SecurityMiddleware SECURE_BROWSER_XSS_FILTER = True @@ -553,6 +559,7 @@ def skip_unreadable_post(record): #SECURE_SSL_REDIRECT = True # Relax the COOP policy to allow Meetecho authentication pop-up SECURE_CROSS_ORIGIN_OPENER_POLICY = "unsafe-none" +SECURE_REFERRER_POLICY = "strict-origin-when-cross-origin" # Override this in your settings_local with the IP addresses relevant for you: INTERNAL_IPS = ( @@ -635,13 +642,9 @@ def skip_unreadable_post(record): IDTRACKER_BASE_URL = "https://datatracker.ietf.org" RFCDIFF_BASE_URL = "https://author-tools.ietf.org/iddiff" IDNITS_BASE_URL = "https://author-tools.ietf.org/api/idnits" +IDNITS3_BASE_URL = "https://author-tools.ietf.org/idnits3/results" IDNITS_SERVICE_URL = "https://author-tools.ietf.org/idnits" -# Content security policy configuration (django-csp) -# (In current production, the Content-Security-Policy header is completely set by nginx configuration, but -# we try to keep this in sync to avoid confusion) -CSP_DEFAULT_SRC = ("'self'", "'unsafe-inline'", f"data: {IDTRACKER_BASE_URL} http://ietf.org/ https://www.ietf.org/ https://analytics.ietf.org/ https://static.ietf.org") - # The name of the method to use to invoke the test suite TEST_RUNNER = 'ietf.utils.test_runner.IetfTestRunner' @@ -680,6 +683,7 @@ def skip_unreadable_post(record): "ietf/utils/patch.py", "ietf/utils/test_data.py", "ietf/utils/jstest.py", + "ietf/utils/coverage.py", ] # These are code line regex patterns @@ -693,12 +697,15 @@ def skip_unreadable_post(record): ] # These are filename globs. They are used by test_parse_templates() and -# get_template_paths() +# get_template_paths(). Globs are applied via pathlib.Path().match, using +# the path to the template from the project root. TEST_TEMPLATE_IGNORE = [ - ".*", # dot-files - "*~", # tilde temp-files - "#*", # files beginning with a hashmark - "500.html" # isn't loaded by regular loader, but checked by test_500_page() + ".*", # dot-files + "*~", # tilde temp-files + "#*", # files beginning with a hashmark + "500.html", # isn't loaded by regular loader, but checked by test_500_page() + "ietf/templates/admin/meeting/RegistrationTicket/change_list.html", + "ietf/templates/admin/meeting/Registration/change_list.html", ] TEST_COVERAGE_MAIN_FILE = os.path.join(BASE_DIR, "../release-coverage.json") @@ -706,8 +713,8 @@ def skip_unreadable_post(record): TEST_CODE_COVERAGE_CHECKER = None if SERVER_MODE != 'production': - import coverage - TEST_CODE_COVERAGE_CHECKER = coverage.Coverage(source=[ BASE_DIR ], cover_pylib=False, omit=TEST_CODE_COVERAGE_EXCLUDE_FILES) + from ietf.utils.coverage import CoverageManager + TEST_CODE_COVERAGE_CHECKER = CoverageManager() TEST_CODE_COVERAGE_REPORT_PATH = "coverage/" TEST_CODE_COVERAGE_REPORT_URL = os.path.join(STATIC_URL, TEST_CODE_COVERAGE_REPORT_PATH, "index.html") @@ -748,33 +755,58 @@ def skip_unreadable_post(record): "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}, } -# settings_local will need to configure storages for these names -MORE_STORAGE_NAMES: list[str] = [ +# Storages for artifacts stored as blobs +ARTIFACT_STORAGE_NAMES: list[str] = [ + "active-draft", + "agenda", + "bibxml-ids", + "bluesheets", "bofreq", "charter", + "chatlog", "conflrev", - "active-draft", "draft", - "slides", + "floorplan", + "indexes", + "liai-att", + "meetinghostlogo", "minutes", - "agenda", - "bluesheets", - "procmaterials", "narrativeminutes", - "statement", + "photo", + "polls", + "procmaterials", + "review", + "rfc", + "slides", + "staging", "statchg", - "liai-att", + "statement", +] +for storagename in ARTIFACT_STORAGE_NAMES: + STORAGES[storagename] = { + "BACKEND": "ietf.doc.storage.StoredObjectBlobdbStorage", + "OPTIONS": {"bucket_name": storagename}, + } + +# Buckets / doc types of meeting materials the CF worker is allowed to serve. This +# differs from the list in Session.meeting_related() by the omission of "recording" +MATERIALS_TYPES_SERVED_BY_WORKER = [ + "agenda", + "bluesheets", "chatlog", + "minutes", + "narrativeminutes", "polls", - "staging", - "bibxml-ids", - "indexes", - "floorplan", - "meetinghostlogo", - "photo", - "review", + "procmaterials", + "slides", ] +# Other storages +STORAGES["red_bucket"] = { + "BACKEND": "django.core.files.storage.InMemoryStorage", + "OPTIONS": {"location": "red_bucket"}, +} + # Override this in settings_local.py if needed # *_PATH variables ends with a slash/ . @@ -808,6 +840,8 @@ def skip_unreadable_post(record): # Mailing list info URL for lists hosted on the IETF servers MAILING_LIST_INFO_URL = "https://mailman3.%(domain)s/mailman3/lists/%(list_addr)s.%(domain)s" MAILING_LIST_ARCHIVE_URL = "https://mailarchive.ietf.org" +MAILING_LIST_ARCHIVE_SEARCH_URL = "https://mailarchive.ietf.org/api/v1/message/search/" +MAILING_LIST_ARCHIVE_API_KEY = "changeme" # Liaison Statement Tool settings (one is used in DOC_HREFS below) LIAISON_UNIVERSAL_FROM = 'Liaison Statement Management Tool ' @@ -859,16 +893,15 @@ def skip_unreadable_post(record): IANA_SYNC_CHANGES_URL = "https://datatracker.iana.org:4443/data-tracker/changes" IANA_SYNC_PROTOCOLS_URL = "https://www.iana.org/protocols/" -RFC_EDITOR_SYNC_PASSWORD="secret" -RFC_EDITOR_SYNC_NOTIFICATION_URL = "https://www.rfc-editor.org/parser/parser.php" RFC_EDITOR_GROUP_NOTIFICATION_EMAIL = "webmaster@rfc-editor.org" -#RFC_EDITOR_GROUP_NOTIFICATION_URL = "https://www.rfc-editor.org/notification/group.php" RFC_EDITOR_QUEUE_URL = "https://www.rfc-editor.org/queue2.xml" RFC_EDITOR_INDEX_URL = "https://www.rfc-editor.org/rfc/rfc-index.xml" RFC_EDITOR_ERRATA_JSON_URL = "https://www.rfc-editor.org/errata.json" -RFC_EDITOR_ERRATA_URL = "https://www.rfc-editor.org/errata_search.php?rfc={rfc_number}" RFC_EDITOR_INLINE_ERRATA_URL = "https://www.rfc-editor.org/rfc/inline-errata/rfc{rfc_number}.html" +RFC_EDITOR_ERRATA_BASE_URL = "https://www.rfc-editor.org/errata/" RFC_EDITOR_INFO_BASE_URL = "https://www.rfc-editor.org/info/" +RFC_EDITOR_QUEUE_SITE_BASE_URL = "https://queue.rfc-editor.org" + # NomCom Tool settings ROLODEX_URL = "" @@ -1111,7 +1144,9 @@ def skip_unreadable_post(record): "--outdir" ] -STATS_REGISTRATION_ATTENDEES_JSON_URL = 'https://registration.ietf.org/{number}/attendees/' +REGISTRATION_PARTICIPANTS_API_URL = 'https://registration.ietf.org/api/v1/participants-dt/' +REGISTRATION_PARTICIPANTS_API_KEY = 'changeme' + PROCEEDINGS_VERSION_CHANGES = [ 0, # version 1 97, # version 2: meeting 97 and later (was number was NEW_PROCEEDINGS_START) @@ -1224,7 +1259,10 @@ def skip_unreadable_post(record): 'patch/change-oidc-provider-field-sizes-228.patch', 'patch/fix-oidc-access-token-post.patch', 'patch/fix-jwkest-jwt-logging.patch', - 'patch/django-cookie-delete-with-all-settings.patch', + # Patch includes old cookie-delete-with-all-settings and a backport of the fix + # to CVE-2026-35192 from Django 5.2. The patches conflict, so cannot be applied + # separately. + 'patch/django-cookie-delete-settings-and-CVE-2026-35192.patch', 'patch/tastypie-django22-fielderror-response.patch', ] if DEBUG: @@ -1234,7 +1272,7 @@ def skip_unreadable_post(record): except ImportError: pass -STATS_NAMES_LIMIT = 25 +STATS_TIMELINE_CACHE_TIMEOUT = 86400 UTILS_MEETING_CONFERENCE_DOMAINS = ['webex.com', 'zoom.us', 'jitsi.org', 'meetecho.com', 'gather.town', ] UTILS_TEST_RANDOM_STATE_FILE = '.factoryboy_random_state' @@ -1266,6 +1304,7 @@ def skip_unreadable_post(record): CELERY_BROKER_URL = 'amqp://mq/' CELERY_BEAT_SCHEDULER = 'django_celery_beat.schedulers:DatabaseScheduler' CELERY_BEAT_SYNC_EVERY = 1 # update DB after every event +CELERY_BEAT_CRON_STARTING_DEADLINE = 1800 # seconds after a missed deadline before abandoning a cron task CELERY_BROKER_CONNECTION_RETRY_ON_STARTUP = True # the default, but setting it squelches a warning # Use a result backend so we can chain tasks. This uses the rpc backend, see # https://docs.celeryq.dev/en/stable/userguide/tasks.html#rpc-result-backend-rabbitmq-qpid @@ -1275,6 +1314,9 @@ def skip_unreadable_post(record): CELERY_CACHE_BACKEND = 'celery-results' # which Django cache to use CELERY_RESULT_EXPIRES = datetime.timedelta(minutes=5) # how long are results valid? (Default is 1 day) CELERY_TASK_IGNORE_RESULT = True # ignore results unless specifically enabled for a task +CELERY_TASK_ROUTES = { + "ietf.blobdb.tasks.pybob_the_blob_replicator_task": {"queue": "blobdb"} +} # Meetecho API setup: Uncomment this and provide real credentials to enable # Meetecho conference creation for interim session requests @@ -1296,6 +1338,11 @@ def skip_unreadable_post(record): MEETECHO_AUDIO_STREAM_URL = "https://mp3.conf.meetecho.com/ietf{session.meeting.number}/{session.pk}.m3u" MEETECHO_SESSION_RECORDING_URL = "https://meetecho-player.ietf.org/playout/?session={session_label}" +# Errata system api configuration +# settings should provide +# ERRATA_METADATA_NOTIFICATION_URL +# ERRATA_METADATA_NOTIFICATION_API_KEY + # Put the production SECRET_KEY in settings_local.py, and also any other # sensitive or site-specific changes. DO NOT commit settings_local.py to svn. from ietf.settings_local import * # pyflakes:ignore pylint: disable=wildcard-import @@ -1325,6 +1372,27 @@ def skip_unreadable_post(record): "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", "VERSION": __version__, "KEY_PREFIX": "ietf:dt", + # Key function is default except with sha384-encoded key + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, + "agenda": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt:agenda", + # Key function is default except with sha384-encoded key + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, + "proceedings": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt:proceedings", + # Key function is default except with sha384-encoded key "KEY_FUNCTION": lambda key, key_prefix, version: ( f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" ), @@ -1372,6 +1440,28 @@ def skip_unreadable_post(record): "VERSION": __version__, "KEY_PREFIX": "ietf:dt", }, + "agenda": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", + # "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + # "LOCATION": "127.0.0.1:11211", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt:agenda", + # Key function is default except with sha384-encoded key + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, + "proceedings": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", + # "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + # "LOCATION": "127.0.0.1:11211", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt:proceedings", + # Key function is default except with sha384-encoded key + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, "sessions": { "BACKEND": "django.core.cache.backends.locmem.LocMemCache", }, @@ -1427,11 +1517,17 @@ def skip_unreadable_post(record): NOMCOM_APP_SECRET = b'\x9b\xdas1\xec\xd5\xa0SI~\xcb\xd4\xf5t\x99\xc4i\xd7\x9f\x0b\xa9\xe8\xfeY\x80$\x1e\x12tN:\x84' ALLOWED_HOSTS = ['*',] - + try: # see https://github.com/omarish/django-cprofile-middleware - import django_cprofile_middleware # pyflakes:ignore - MIDDLEWARE = MIDDLEWARE + ['django_cprofile_middleware.middleware.ProfilerMiddleware', ] + import django_cprofile_middleware # pyflakes:ignore + + MIDDLEWARE = MIDDLEWARE + [ + "django_cprofile_middleware.middleware.ProfilerMiddleware", + ] + DJANGO_CPROFILE_MIDDLEWARE_REQUIRE_STAFF = ( + False # Do not use this setting for a public site! + ) except ImportError: pass @@ -1444,3 +1540,5 @@ def skip_unreadable_post(record): YOUTUBE_DOMAINS = ['www.youtube.com', 'youtube.com', 'youtu.be', 'm.youtube.com', 'youtube-nocookie.com', 'www.youtube-nocookie.com'] + +IETF_DOI_PREFIX = "10.17487" diff --git a/ietf/settings_test.py b/ietf/settings_test.py index fe77152d42..e7ebc13eb2 100755 --- a/ietf/settings_test.py +++ b/ietf/settings_test.py @@ -14,8 +14,7 @@ import shutil import tempfile from ietf.settings import * # pyflakes:ignore -from ietf.settings import STORAGES, TEST_CODE_COVERAGE_CHECKER, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS -import botocore.config +from ietf.settings import ORIG_AUTH_PASSWORD_VALIDATORS, STORAGES import debug # pyflakes:ignore debug.debug = True @@ -49,9 +48,12 @@ def __getitem__(self, item): }, } -if TEST_CODE_COVERAGE_CHECKER and not TEST_CODE_COVERAGE_CHECKER._started: # pyflakes:ignore - TEST_CODE_COVERAGE_CHECKER.start() # pyflakes:ignore +# test with a single DB - do not use a DB router +BLOBDB_DATABASE = "default" +DATABASE_ROUTERS = [] # type: ignore +if TEST_CODE_COVERAGE_CHECKER: # pyflakes:ignore + TEST_CODE_COVERAGE_CHECKER.start() # pyflakes:ignore def tempdir_with_cleanup(**kwargs): """Utility to create a temporary dir and arrange cleanup""" @@ -107,29 +109,18 @@ def tempdir_with_cleanup(**kwargs): }, } -# Configure storages for the blob store - use env settings if present. See the --no-manage-blobstore test option. -_blob_store_endpoint_url = os.environ.get("DATATRACKER_BLOB_STORE_ENDPOINT_URL", "http://blobstore:9000") -_blob_store_access_key = os.environ.get("DATATRACKER_BLOB_STORE_ACCESS_KEY", "minio_root") -_blob_store_secret_key = os.environ.get("DATATRACKER_BLOB_STORE_SECRET_KEY", "minio_pass") -_blob_store_bucket_prefix = os.environ.get("DATATRACKER_BLOB_STORE_BUCKET_PREFIX", "test-") -_blob_store_enable_profiling = ( - os.environ.get("DATATRACKER_BLOB_STORE_ENABLE_PROFILING", "false").lower() == "true" -) -for storagename in MORE_STORAGE_NAMES: - STORAGES[storagename] = { - "BACKEND": "ietf.doc.storage_backends.CustomS3Storage", - "OPTIONS": dict( - endpoint_url=_blob_store_endpoint_url, - access_key=_blob_store_access_key, - secret_key=_blob_store_secret_key, - security_token=None, - client_config=botocore.config.Config( - signature_version="s3v4", - connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT, - read_timeout=BLOBSTORAGE_READ_TIMEOUT, - retries={"total_max_attempts": BLOBSTORAGE_MAX_ATTEMPTS}, - ), - bucket_name=f"{_blob_store_bucket_prefix}{storagename}", - ietf_log_blob_timing=_blob_store_enable_profiling, - ), - } +# Restore AUTH_PASSWORD_VALIDATORS if they were reset in settings_local +try: + AUTH_PASSWORD_VALIDATORS = ORIG_AUTH_PASSWORD_VALIDATORS +except NameError: + pass + +# Use InMemoryStorage for red bucket and r2-rfc storages +STORAGES["red_bucket"] = { + "BACKEND": "django.core.files.storage.InMemoryStorage", + "OPTIONS": {"location": "red_bucket"}, +} +STORAGES["r2-rfc"] = { + "BACKEND": "django.core.files.storage.InMemoryStorage", + "OPTIONS": {"location": "r2-rfc"}, +} diff --git a/ietf/settings_testcrawl.py b/ietf/settings_testcrawl.py index a1b5ce8946..edb978757a 100644 --- a/ietf/settings_testcrawl.py +++ b/ietf/settings_testcrawl.py @@ -27,9 +27,14 @@ 'MAX_ENTRIES': 10000, }, }, + 'agenda': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + }, + 'proceedings': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + }, 'sessions': { 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - # No version-specific VERSION setting. }, 'htmlized': { 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', diff --git a/ietf/static/css/datepicker.scss b/ietf/static/css/datepicker.scss index 88f9e835fd..b193ccda3a 100644 --- a/ietf/static/css/datepicker.scss +++ b/ietf/static/css/datepicker.scss @@ -4,3 +4,29 @@ $dp-cell-focus-background-color: $dropdown-link-hover-bg !default; @import "vanillajs-datepicker/sass/datepicker-bs5"; + +[data-bs-theme="dark"] .datepicker-picker { + .datepicker-header, + .datepicker-controls .btn, + .datepicker-main, + .datepicker-footer { + background-color: $gray-800; + } + + .datepicker-cell:hover { + background-color: $gray-700; + } + + .datepicker-cell.day.focused { + background-color: $gray-600; + } + + .datepicker-cell.day.selected.focused { + background-color: $blue; + } + + .datepicker-controls .btn:hover { + background-color:$gray-700; + color: $gray-400; + } +} diff --git a/ietf/static/css/ietf.scss b/ietf/static/css/ietf.scss index aee93dcfe1..b8c701eae1 100644 --- a/ietf/static/css/ietf.scss +++ b/ietf/static/css/ietf.scss @@ -46,6 +46,11 @@ $bootstrap-icons-font-src: url("npm:bootstrap-icons/font/fonts/bootstrap-icons.w url("npm:bootstrap-icons/font/fonts/bootstrap-icons.woff") format("woff"); @import "bootstrap-icons/font/bootstrap-icons"; +// Disable contextual alternates (calt) +body { + font-feature-settings: "calt" off; +} + // Leave room for fixed-top navbar... body.navbar-offset { padding-top: 60px; @@ -229,7 +234,7 @@ th, .group-menu .dropdown-menu { height: auto; width: auto; - max-height: 35em; + max-height: 95vh; overflow-x: hidden; overflow-y: auto; } @@ -510,23 +515,23 @@ td.position-empty { tr.position-notready-row, tr.position-discuss-row, tr.position-block-row { - background-color: shade-color($color-discuss, 85%); + background-color: shade-color($color-discuss, 65%); } tr.position-yes-row { - background-color: shade-color($color-yes, 75%); + background-color: shade-color($color-yes, 65%); } tr.position-noobj-row { - background-color: shade-color($color-noobj, 75%); + background-color: shade-color($color-noobj, 65%); } tr.position-abstain-row { - background-color: shade-color($color-abstain, 85%); + background-color: shade-color($color-abstain, 65%); } tr.position-recuse-row { - background-color: shade-color($color-recuse, 85%); + background-color: shade-color($color-recuse, 65%); } } @@ -1216,3 +1221,20 @@ iframe.status { .overflow-shadows--bottom-only { box-shadow: inset 0px -21px 18px -20px var(--bs-body-color); } + +#navbar-doc-search-wrapper { + position: relative; +} + +#navbar-doc-search-results { + max-height: 400px; + overflow-y: auto; + min-width: auto; + left: 0; + right: 0; + + .dropdown-item { + white-space: normal; + overflow-wrap: break-word; + } +} diff --git a/ietf/static/js/announcement.js b/ietf/static/js/announcement.js new file mode 100644 index 0000000000..95465120fa --- /dev/null +++ b/ietf/static/js/announcement.js @@ -0,0 +1,57 @@ +const announcementApp = (function() { + 'use strict'; + return { + // functions for Announcement + checkToField: function() { + document.documentElement.scrollTop = 0; // For most browsers + const toField = document.getElementById('id_to'); + const toCustomInput = document.getElementById('id_to_custom'); + const toCustomDiv = toCustomInput.closest('div.row'); + + if (toField.value === 'Other...') { + toCustomDiv.style.display = 'flex'; // Show the custom field + } else { + toCustomDiv.style.display = 'none'; // Hide the custom field + toCustomInput.value = ''; // Optionally clear the input value if hidden + } + } + }; +})(); + +// Extra care is required to ensure the back button +// works properly for the optional to_custom field. +// Take the case when a user selects "Other..." for +// "To" field. The "To custom" field appears and they +// enter a new address there. +// In Chrome, when the form is submitted and then the user +// uses the back button (or browser back), the page loads +// from bfcache then the javascript DOMContentLoaded event +// handler is run, hiding the empty to_custom field, THEN the +// browser autofills the form fields. Because to_submit +// is now hidden it does not get a value. This is a very +// bad experience for the user because the to_custom field +// was unexpectedly cleared and hidden. If they notice this +// they would need to know to first select another "To" +// option, then select "Other..." again just to get the +// to_custom field visible so they can re-enter the custom +// address. +// The solution is to use setTimeout to run checkToField +// after a short delay, giving the browser time to autofill +// the form fields before it checks to see if the to_custom +// field is empty and hides it. + +document.addEventListener('DOMContentLoaded', function() { + // Run the visibility check after allowing cache to populate values + setTimeout(announcementApp.checkToField, 300); + + const toField = document.getElementById('id_to'); + toField.addEventListener('change', announcementApp.checkToField); +}); + +// Handle back/forward navigation with pageshow +window.addEventListener('pageshow', function(event) { + if (event.persisted) { + // Then apply visibility logic after cache restoration + setTimeout(announcementApp.checkToField, 300); + } +}); \ No newline at end of file diff --git a/ietf/static/js/attendees-chart.js b/ietf/static/js/attendees-chart.js new file mode 100644 index 0000000000..fed3b1289c --- /dev/null +++ b/ietf/static/js/attendees-chart.js @@ -0,0 +1,58 @@ +(function () { + var raw = document.getElementById('attendees-chart-data'); + if (!raw) return; + var chartData = JSON.parse(raw.textContent); + var chart = null; + var currentBreakdown = 'type'; + + // Override the global transparent background set by highcharts.js so the + // export menu and fullscreen view use the page background color. + var container = document.getElementById('attendees-pie-chart'); + var bodyBg = getComputedStyle(document.body).backgroundColor; + container.style.setProperty('--highcharts-background-color', bodyBg); + + function renderChart(breakdown) { + var seriesData = chartData[breakdown].map(function (item) { + return { name: item[0], y: item[1] }; + }); + if (chart) chart.destroy(); + chart = Highcharts.chart(container, { + chart: { type: 'pie', height: 400 }, + title: { text: null }, + tooltip: { pointFormat: '{point.name}: {point.y} ({point.percentage:.1f}%)' }, + plotOptions: { + pie: { + dataLabels: { + enabled: true, + format: '{point.name}
{point.y} ({point.percentage:.1f}%)', + }, + showInLegend: false, + } + }, + series: [{ name: 'Attendees', data: seriesData }], + }); + } + + var modal = document.getElementById('attendees-chart-modal'); + + // Render (or re-render) the chart each time the modal becomes fully visible, + // so Highcharts can measure the container dimensions correctly. + modal.addEventListener('shown.bs.modal', function () { + renderChart(currentBreakdown); + }); + + // Release the chart when the modal closes to avoid stale renders. + modal.addEventListener('hidden.bs.modal', function () { + if (chart) { + chart.destroy(); + chart = null; + } + }); + + document.querySelectorAll('[name="attendees-breakdown"]').forEach(function (radio) { + radio.addEventListener('change', function () { + currentBreakdown = this.value; + renderChart(currentBreakdown); + }); + }); +})(); diff --git a/ietf/static/js/custom_striped.js b/ietf/static/js/custom_striped.js new file mode 100644 index 0000000000..480ad7cf82 --- /dev/null +++ b/ietf/static/js/custom_striped.js @@ -0,0 +1,16 @@ +// Copyright The IETF Trust 2025, All Rights Reserved + +document.addEventListener('DOMContentLoaded', () => { + // add stripes + const firstRow = document.querySelector('.custom-stripe .row') + if (firstRow) { + const parent = firstRow.parentElement; + const allRows = Array.from(parent.children).filter(child => child.classList.contains('row')) + allRows.forEach((row, index) => { + row.classList.remove('bg-light') + if (index % 2 === 1) { + row.classList.add('bg-light') + } + }) + } +}) diff --git a/ietf/static/js/document_html.js b/ietf/static/js/document_html.js index 6e8861739a..3e609f3965 100644 --- a/ietf/static/js/document_html.js +++ b/ietf/static/js/document_html.js @@ -117,4 +117,83 @@ document.addEventListener("DOMContentLoaded", function (event) { } }); } + + // Rewrite these CSS properties so that the values are available for restyling. + document.querySelectorAll("svg [style]").forEach(el => { + // Push these CSS properties into their own attributes + const SVG_PRESENTATION_ATTRS = new Set([ + 'alignment-baseline', 'baseline-shift', 'clip', 'clip-path', 'clip-rule', + 'color', 'color-interpolation', 'color-interpolation-filters', + 'color-rendering', 'cursor', 'direction', 'display', 'dominant-baseline', + 'fill', 'fill-opacity', 'fill-rule', 'filter', 'flood-color', + 'flood-opacity', 'font-family', 'font-size', 'font-size-adjust', + 'font-stretch', 'font-style', 'font-variant', 'font-weight', + 'image-rendering', 'letter-spacing', 'lighting-color', 'marker-end', + 'marker-mid', 'marker-start', 'mask', 'opacity', 'overflow', 'paint-order', + 'pointer-events', 'shape-rendering', 'stop-color', 'stop-opacity', + 'stroke', 'stroke-dasharray', 'stroke-dashoffset', 'stroke-linecap', + 'stroke-linejoin', 'stroke-miterlimit', 'stroke-opacity', 'stroke-width', + 'text-anchor', 'text-decoration', 'text-rendering', 'unicode-bidi', + 'vector-effect', 'visibility', 'word-spacing', 'writing-mode', + ]); + + // Simple CSS splitter: respects quoted strings and parens so semicolons + // inside url(...) or "..." don't get treated as declaration boundaries. + function parseDeclarations(styleText) { + const decls = []; + let buf = ''; + let inStr = false; + let strChar = ''; + let escaped = false; + let depth = 0; + + for (const ch of styleText) { + if (inStr) { + if (escaped) { + escaped = false; + } else if (ch === '\\') { + escaped = true; + } else if (ch === strChar) { + inStr = false; + } + } else if (ch === '"' || ch === "'") { + inStr = true; + strChar = ch; + } else if (ch === '(') { + depth++; + } else if (ch === ')') { + depth--; + } else if (ch === ';' && depth === 0) { + const trimmed = buf.trim(); + if (trimmed) { + decls.push(trimmed); + } + buf = ''; + continue; + } + buf += ch; + } + const trimmed = buf.trim(); + if (trimmed) { + decls.push(trimmed); + } + return decls; + } + + const remainder = []; + for (const decl of parseDeclarations(el.getAttribute('style'))) { + const [prop, val] = decl.split(":", 2).map(v => v.trim()); + if (val && !/!important$/.test(val) && SVG_PRESENTATION_ATTRS.has(prop)) { + el.setAttribute(prop, val); + } else { + remainder.push(decl); + } + } + + if (remainder.length > 0) { + el.setAttribute('style', remainder.join('; ')); + } else { + el.removeAttribute('style'); + } + }); }); diff --git a/ietf/static/js/edit-meeting-schedule.js b/ietf/static/js/edit-meeting-schedule.js index 3f0afe0841..2a73a8c29d 100644 --- a/ietf/static/js/edit-meeting-schedule.js +++ b/ietf/static/js/edit-meeting-schedule.js @@ -50,6 +50,7 @@ $(function () { let sessionPurposeInputs = schedEditor.find('.session-purpose-toggles input'); let timeSlotGroupInputs = schedEditor.find("#timeslot-group-toggles-modal .modal-body .individual-timeslots input"); let sessionParentInputs = schedEditor.find(".session-parent-toggles input"); + let sessionParentToggleAll = schedEditor.find(".session-parent-toggles .session-parent-toggle-all") const classes_to_hide = '.hidden-timeslot-group,.hidden-timeslot-type'; // hack to work around lack of position sticky support in old browsers, see https://caniuse.com/#feat=css-sticky @@ -769,6 +770,17 @@ $(function () { sessionParentInputs.on("click", updateSessionParentToggling); updateSessionParentToggling(); + // Toggle _all_ session parents + function toggleAllSessionParents() { + if (sessionParentInputs.filter(":checked").length < sessionParentInputs.length) { + sessionParentInputs.prop("checked", true); + } else { + sessionParentInputs.prop("checked", false); + } + updateSessionParentToggling(); + } + sessionParentToggleAll.on("click", toggleAllSessionParents); + // Toggling timeslot types function updateTimeSlotTypeToggling() { const checkedTypes = jQuery.map(timeSlotTypeInputs.filter(":checked"), elt => elt.value); @@ -1020,4 +1032,4 @@ $(function () { .on("mouseleave", ".other-session", function () { sessions.filter("#session" + this.dataset.othersessionid).removeClass("highlight"); }); -}); \ No newline at end of file +}); diff --git a/ietf/static/js/ietf.js b/ietf/static/js/ietf.js index dde00c6d1e..09fa324e42 100644 --- a/ietf/static/js/ietf.js +++ b/ietf/static/js/ietf.js @@ -13,7 +13,7 @@ import "bootstrap/js/dist/scrollspy"; import "bootstrap/js/dist/tab"; // import "bootstrap/js/dist/toast"; import "bootstrap/js/dist/tooltip"; - +import { debounce } from 'lodash-es'; import jquery from "jquery"; window.$ = window.jQuery = jquery; @@ -112,6 +112,34 @@ function overflowShadows(el) { } } +function ensureDropdownOnscreen(elm) { + const handlePlacement = () => { + if(!(elm instanceof HTMLElement)) { + return + } + const rect = elm.getBoundingClientRect() + const BUFFER_PX = 5 // additional distance from bottom of viewport + const existingStyleTop = parseInt(elm.style.top, 10) + const offscreenBy = Math.round(window.innerHeight - (rect.top + rect.height) - BUFFER_PX) + if(existingStyleTop === offscreenBy) { + console.log(`Already set top to ${offscreenBy}. Ignoring`) + // already set, nothing to do + return + } + if(offscreenBy < 0) { + elm.style.top = `${offscreenBy}px` + } + } + + const debouncedHandler = debounce(handlePlacement, 100) + + const observer = new MutationObserver(debouncedHandler) + + observer.observe(elm, { + attributes: true + }) +} + $(document) .ready(function () { // load data for the menu @@ -144,12 +172,17 @@ $(document) g.acronym + " — " + g.name + ""); } menu.push(""); - for (i = 0; i < attachTo.length; i++) { - attachTo.closest(".dropdown-menu"); - } + attachTo.append(menu.join("")); - attachTo.find(".overflow-shadows").each(function(){ overflowShadows(this)}) + attachTo.find(".overflow-shadows").each(function(){ + // needs to be a function(){} so that we can access jQuery's `this` + overflowShadows(this) + }) + attachTo.find(".dropdown-menu").each(function(){ + // needs to be a function(){} so that we can access jQuery's `this` + ensureDropdownOnscreen(this) + }) } } }); diff --git a/ietf/static/js/liaisons.js b/ietf/static/js/liaisons.js index e7a76660c9..6c46cb6dfc 100644 --- a/ietf/static/js/liaisons.js +++ b/ietf/static/js/liaisons.js @@ -197,16 +197,6 @@ var liaisonForm = { } }, - checkPostOnly: function (post_only) { - if (post_only) { - $("button[name=send]") - .hide(); - } else { - $("button[name=send]") - .show(); - } - }, - updateInfo: function (first_time, sender) { // don't overwrite fields when editing existing liaison if (liaisonForm.is_edit_form) { @@ -239,7 +229,6 @@ var liaisonForm = { liaisonForm.toggleApproval(response.needs_approval); liaisonForm.response_contacts.val(response.response_contacts); } - liaisonForm.checkPostOnly(response.post_only); } } }); @@ -326,4 +315,4 @@ $(document) .each(liaisonForm.init); $('#liaison_search_form') .each(searchForm.init); - }); \ No newline at end of file + }); diff --git a/ietf/static/js/list.js b/ietf/static/js/list.js index c16111ba63..c03368cd72 100644 --- a/ietf/static/js/list.js +++ b/ietf/static/js/list.js @@ -16,7 +16,7 @@ function text_sort(a, b, options) { // sort by text content return prep(a, options).localeCompare(prep(b, options), "en", { sensitivity: "base", - ignorePunctuation: true, + ignorePunctuation: false, numeric: true }); } @@ -247,7 +247,8 @@ $(document) $(table) .find(".sort") - .on("click", function () { + .on("click", function (ev) { + ev.preventDefault() var order = $(this) .hasClass("asc") ? "desc" : "asc"; $.each(list_instance, (_, e) => { diff --git a/ietf/static/js/meeting_stats.js b/ietf/static/js/meeting_stats.js new file mode 100644 index 0000000000..cf43d08eb8 --- /dev/null +++ b/ietf/static/js/meeting_stats.js @@ -0,0 +1,58 @@ +// Copyright The IETF Trust 2026, All Rights Reserved +import Chart from 'chart.js/auto' +import autocolors from 'chartjs-plugin-autocolors' + +document.addEventListener('DOMContentLoaded', () => { + Chart.register(autocolors) + // ── Safely parse JSON data injected from Django view ── + const totalChartData = JSON.parse(document.getElementById('total-chart-data').textContent) + const inPersonChartData = JSON.parse(document.getElementById('in-person-chart-data').textContent) + + function displayChart (id, data) { + const ctx = document.getElementById(id).getContext('2d') + new Chart(ctx, { + type: 'pie', // Change to 'doughnut' for a donut chart + data: data, + options: { + responsive: true, + plugins: { + autocolors: { + mode: 'data' // Required for Pie charts to color individual slices + }, + legend: { + position: 'bottom', + labels: { + padding: 20, + font: { size: 13 }, + color: '#475569', + generateLabels: function (chart) { + const dataset = chart.data.datasets[0] + return chart.data.labels.map((label, i) => ({ + text: `${label}: ${dataset.data[i]}`, + fillStyle: dataset.backgroundColor[i], + hidden: false, + index: i, + })) + } + } + }, + tooltip: { + callbacks: { + label: function (context) { + const label = context.label || '' + const value = context.raw + const total = context.dataset.data.reduce((a, b) => a + b, 0) + const percentage = ((value / total) * 100).toFixed(1) + + return `${label}: ${value} (${percentage}%)` + } + } + } + } + } + }) + } + + displayChart('totalRegistrationChart', totalChartData) + displayChart('inPersonRegistrationChart', inPersonChartData) +}) diff --git a/ietf/static/js/meeting_timeline.js b/ietf/static/js/meeting_timeline.js new file mode 100644 index 0000000000..713fb3ae70 --- /dev/null +++ b/ietf/static/js/meeting_timeline.js @@ -0,0 +1,92 @@ +// Copyright The IETF Trust 2026, All Rights Reserved +import Chart from 'chart.js/auto' +import zoomPlugin from 'chartjs-plugin-zoom' + +document.addEventListener('DOMContentLoaded', () => { + Chart.register(zoomPlugin) // enable the zoom plugin + + // ── Safely parse JSON data injected from Django view ── + const totalChartData = JSON.parse(document.getElementById('total-chart-data').textContent) + const inPersonChartData = JSON.parse(document.getElementById('in-person-chart-data').textContent) + const statsType = JSON.parse(document.getElementById('stats-type-data').textContent) + const stackedLines = statsType === 'total' + + function displayChart (id, data) { + const ctx = document.getElementById(id).getContext('2d') + return new Chart(ctx, { + type: 'line', // Change to 'doughnut' for a donut chart + data: data, + options: { + responsive: true, + scales: { + y: { + stacked: stackedLines, + }, + x: { + title: { + display: true, + text: 'IETF Meeting Number', + }, + }, + }, + plugins: { + legend: { + position: 'bottom', + labels: { + usePointStyle: true, + padding: 15, + font: { size: 12 }, + }, + }, + tooltip: { + backgroundColor: 'rgba(0,0,0,0.8)', + titleFont: { size: 14 }, + bodyFont: { size: 13 }, + callbacks: { + title: function (items) { + return `IETF Meeting ${items[0].label}` + }, + label: function (context) { + return ` ${context.dataset.label}: ${context.parsed.y} participants` + } + } + }, + zoom: { + zoom: { + wheel: { enabled: true }, // scroll to zoom + pinch: { enabled: true }, // pinch on mobile + drag: { // drag to select range + enabled: true, + modifierKey: 'alt' + }, + mode: 'xy', // zoom X-axis and Y-axis + }, + pan: { + enabled: true, + mode: 'xy', // pan X-axis and Y-axis + }, + }, + } + } + }) + } + + const totalChart = displayChart('totalRegistrationChart', totalChartData) + if (inPersonChartData !== null) { + inPersonChart = displayChart('inPersonRegistrationChart', inPersonChartData) + } + document.addEventListener('keydown', (event) => { + if (event.key === 'Escape') { + totalChart.resetZoom() + if (inPersonChart !== null) { + inPersonChart.resetZoom() + } + } + }) + document.getElementById('resetButton').addEventListener('click', () => { + totalChart.resetZoom() + if (inPersonChart !== null) { + inPersonChart.resetZoom() + } + }) +}) diff --git a/ietf/static/js/navbar-doc-search.js b/ietf/static/js/navbar-doc-search.js new file mode 100644 index 0000000000..c36c032310 --- /dev/null +++ b/ietf/static/js/navbar-doc-search.js @@ -0,0 +1,113 @@ +$(function () { + var $input = $('#navbar-doc-search'); + var $results = $('#navbar-doc-search-results'); + var ajaxUrl = $input.data('ajax-url'); + var debounceTimer = null; + var highlightedIndex = -1; + var keyboardHighlight = false; + var currentItems = []; + + function showDropdown() { + $results.addClass('show'); + } + + function hideDropdown() { + $results.removeClass('show'); + highlightedIndex = -1; + keyboardHighlight = false; + updateHighlight(); + } + + function updateHighlight() { + $results.find('.dropdown-item').removeClass('active'); + if (highlightedIndex >= 0 && highlightedIndex < currentItems.length) { + $results.find('.dropdown-item').eq(highlightedIndex).addClass('active'); + } + } + + function doSearch(query) { + if (query.length < 2) { + hideDropdown(); + return; + } + $.ajax({ + url: ajaxUrl, + dataType: 'json', + data: { q: query }, + success: function (data) { + currentItems = data; + highlightedIndex = -1; + $results.empty(); + if (data.length === 0) { + $results.append('
  • No results found
    • '); + } else { + data.forEach(function (item) { + var $li = $('
  • '); + var $a = $('' + item.text + ''); + $li.append($a); + $results.append($li); + }); + } + showDropdown(); + } + }); + } + + $input.on('input', function () { + clearTimeout(debounceTimer); + var query = $(this).val().trim(); + debounceTimer = setTimeout(function () { + doSearch(query); + }, 250); + }); + + $input.on('keydown', function (e) { + if (e.key === 'ArrowDown') { + e.preventDefault(); + if (highlightedIndex < currentItems.length - 1) { + highlightedIndex++; + keyboardHighlight = true; + updateHighlight(); + } + } else if (e.key === 'ArrowUp') { + e.preventDefault(); + if (highlightedIndex > 0) { + highlightedIndex--; + keyboardHighlight = true; + updateHighlight(); + } + } else if (e.key === 'Enter') { + e.preventDefault(); + if (keyboardHighlight && highlightedIndex >= 0 && highlightedIndex < currentItems.length) { + window.location.href = currentItems[highlightedIndex].url; + } else { + var query = $(this).val().trim(); + if (query) { + window.location.href = '/doc/search/?name=' + encodeURIComponent(query) + '&rfcs=on&activedrafts=on&olddrafts=on'; + } + } + } else if (e.key === 'Escape') { + hideDropdown(); + $input.blur(); + } + }); + + // Hover highlights (visual only — Enter still submits the text) + $results.on('mouseenter', '.dropdown-item', function () { + highlightedIndex = $results.find('.dropdown-item').index(this); + keyboardHighlight = false; + updateHighlight(); + }); + + $results.on('mouseleave', '.dropdown-item', function () { + highlightedIndex = -1; + updateHighlight(); + }); + + // Click outside closes dropdown + $(document).on('click', function (e) { + if (!$(e.target).closest('#navbar-doc-search-wrapper').length) { + hideDropdown(); + } + }); +}); diff --git a/ietf/static/js/password_strength.js b/ietf/static/js/password_strength.js index 4df5c14439..74a245fb5c 100644 --- a/ietf/static/js/password_strength.js +++ b/ietf/static/js/password_strength.js @@ -57,22 +57,40 @@ .parent() .parent() .find('.password_strength_offline_info'); + let password_improvement_hint = $(password_strength_info) + .find('.password_improvement_hint'); if ($(this) .val()) { var result = zxcvbn($(this) .val()); - - if (result.score < 3) { - password_strength_bar.removeClass('text-bg-success') - .addClass('text-bg-warning'); - password_strength_info.find('.badge') - .removeClass('d-none'); + const enforceStrength = !('disableStrengthEnforcement' in this.dataset); + const strongEnough = !enforceStrength || (result.score >= 3); + if (strongEnough) { + // Mark input as valid + this.setCustomValidity(''); } else { + // Mark input as invalid + this.setCustomValidity('This password does not meet complexity requirements'); + } + + if (this.checkValidity()) { password_strength_bar.removeClass('text-bg-warning') .addClass('text-bg-success'); password_strength_info.find('.badge') .addClass('d-none'); + this.classList.remove('is-invalid') + password_improvement_hint.addClass('d-none'); + password_improvement_hint.text('') + } else { + this.classList.add('is-invalid') + password_improvement_hint.text(this.validationMessage) + password_improvement_hint.removeClass('d-none'); + + password_strength_bar.removeClass('text-bg-success') + .addClass('text-bg-warning'); + password_strength_info.find('.badge') + .removeClass('d-none'); } password_strength_bar.width(((result.score + 1) / 5) * 100 + '%') @@ -152,23 +170,31 @@ .data('confirm-with'); if (confirm_with && confirm_with == password_field.attr('id')) { - if (confirm_value && password) { + if (password) { if (confirm_value === password) { $(confirm_field) .parent() .find('.password_strength_info') .addClass('d-none'); + confirm_field.setCustomValidity('') + confirm_field.classList.remove('is-invalid') } else { - $(confirm_field) - .parent() - .find('.password_strength_info') - .removeClass('d-none'); + if (confirm_value !== '') { + $(confirm_field) + .parent() + .find('.password_strength_info') + .removeClass('d-none'); + } + confirm_field.setCustomValidity('Does not match new password') + confirm_field.classList.add('is-invalid') } } else { $(confirm_field) .parent() .find('.password_strength_info') .addClass('d-none'); + confirm_field.setCustomValidity('') + confirm_field.classList.remove('is-invalid') } } }); diff --git a/ietf/static/js/session_details.js b/ietf/static/js/session_details.js new file mode 100644 index 0000000000..03d1b2d3d9 --- /dev/null +++ b/ietf/static/js/session_details.js @@ -0,0 +1,53 @@ +// Copyright The IETF Trust 2026, All Rights Reserved +// Relies on other scripts being loaded, see usage in session_details.html +document.addEventListener('DOMContentLoaded', () => { + // Init with best guess at local timezone. + ietf_timezone.set_tz_change_callback(timezone_changed) // cb is in upcoming.js + ietf_timezone.initialize('local') + + // Set up sortable elements if the user can manage materials + if (document.getElementById('can-manage-materials-flag')) { + const sortables = [] + const options = { + group: 'slides', + animation: 150, + handle: '.drag-handle', + onAdd: function (event) {onAdd(event)}, + onRemove: function (event) {onRemove(event)}, + onEnd: function (event) {onEnd(event)} + } + + function onAdd (event) { + const old_session = event.from.getAttribute('data-session') + const new_session = event.to.getAttribute('data-session') + $.post(event.to.getAttribute('data-add-to-session'), { + 'order': event.newIndex + 1, + 'name': event.item.getAttribute('name') + }) + $(event.item).find('td:eq(1)').find('a').each(function () { + $(this).attr('href', $(this).attr('href').replace(old_session, new_session)) + }) + } + + function onRemove (event) { + const old_session = event.from.getAttribute('data-session') + $.post(event.from.getAttribute('data-remove-from-session'), { + 'oldIndex': event.oldIndex + 1, + 'name': event.item.getAttribute('name') + }) + } + + function onEnd (event) { + if (event.to == event.from) { + $.post(event.from.getAttribute('data-reorder-in-session'), { + 'oldIndex': event.oldIndex + 1, + 'newIndex': event.newIndex + 1 + }) + } + } + + for (const elt of document.querySelectorAll('.slides tbody')) { + sortables.push(Sortable.create(elt, options)) + } + } +}) diff --git a/ietf/secr/static/js/session_form.js b/ietf/static/js/session_form.js similarity index 92% rename from ietf/secr/static/js/session_form.js rename to ietf/static/js/session_form.js index 6f28f16db4..bd61293d7c 100644 --- a/ietf/secr/static/js/session_form.js +++ b/ietf/static/js/session_form.js @@ -1,4 +1,4 @@ -/* Copyright The IETF Trust 2021, All Rights Reserved +/* Copyright The IETF Trust 2021-2025, All Rights Reserved * * JS support for the SessionForm * */ diff --git a/ietf/secr/static/js/sessions.js b/ietf/static/js/session_request.js similarity index 90% rename from ietf/secr/static/js/sessions.js rename to ietf/static/js/session_request.js index a2770e6262..dfb169f675 100644 --- a/ietf/secr/static/js/sessions.js +++ b/ietf/static/js/session_request.js @@ -1,4 +1,4 @@ -// Copyright The IETF Trust 2015-2021, All Rights Reserved +// Copyright The IETF Trust 2015-2025, All Rights Reserved /* global alert */ var ietf_sessions; // public interface @@ -38,7 +38,7 @@ var ietf_sessions; // public interface const only_one_session = (val === 1); if (document.form_post.session_time_relation) { document.form_post.session_time_relation.disabled = only_one_session; - document.form_post.session_time_relation.closest('tr').hidden = only_one_session; + document.form_post.session_time_relation.closest('div.row').hidden = only_one_session; } if (document.form_post.joint_for_session) { document.form_post.joint_for_session.disabled = only_one_session; @@ -129,6 +129,11 @@ var ietf_sessions; // public interface } } + function wg_constraint_delete_clicked(event) { + const constraint_name = event.currentTarget.dataset.constraint_name; + delete_last_wg_constraint(constraint_name); + } + /* Initialization */ function on_load() { // Attach event handler to session count select @@ -146,6 +151,9 @@ var ietf_sessions; // public interface selectors[index].addEventListener('change', wg_constraint_selector_changed, false) } + // Attach event handler to constraint delete buttons + document.querySelectorAll('.wg_constraint_delete') + .forEach(btn => btn.addEventListener('click', wg_constraint_delete_clicked)); } // initialize after page loads diff --git a/ietf/stats/management/commands/fetch_meeting_attendance.py b/ietf/stats/management/commands/fetch_meeting_attendance.py deleted file mode 100644 index e17ae567fa..0000000000 --- a/ietf/stats/management/commands/fetch_meeting_attendance.py +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright The IETF Trust 2017-2019, All Rights Reserved -# Copyright 2016 IETF Trust - -from django.core.management.base import BaseCommand, CommandError -from django.utils import timezone - -import debug # pyflakes:ignore - -from ietf.meeting.models import Meeting -from ietf.stats.utils import fetch_attendance_from_meetings -from ietf.utils import log - - -class Command(BaseCommand): - help = "Fetch meeting attendee figures from ietf.org/registration/attendees." - - def add_arguments(self, parser): - parser.add_argument("--meeting", help="meeting to fetch data for") - parser.add_argument("--all", action="store_true", help="fetch data for all meetings") - parser.add_argument("--latest", type=int, help="fetch data for latest N meetings") - - def handle(self, *args, **options): - self.verbosity = options['verbosity'] - - meetings = Meeting.objects.none() - if options['meeting']: - meetings = Meeting.objects.filter(number=options['meeting'], type="ietf") - elif options['all']: - meetings = Meeting.objects.filter(type="ietf").order_by("date") - elif options['latest']: - meetings = Meeting.objects.filter(type="ietf", date__lte=timezone.now()).order_by("-date")[:options['latest']] - else: - raise CommandError("Please use one of --meeting, --all or --latest") - - for meeting, stats in zip(meetings, fetch_attendance_from_meetings(meetings)): - msg = "Fetched data for meeting {:>3}: {:4d} processed, {:4d} added, {:4d} in table".format( - meeting.number, stats.processed, stats.added, stats.total - ) - if self.stdout.isatty(): - self.stdout.write(msg+'\n') # make debugging a bit easier - else: - log.log(msg) diff --git a/ietf/stats/management/commands/find_meetingregistration_person_issues.py b/ietf/stats/management/commands/find_meetingregistration_person_issues.py deleted file mode 100644 index 4eaf6ac238..0000000000 --- a/ietf/stats/management/commands/find_meetingregistration_person_issues.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright The IETF Trust 2021, All Rights Reserved - -import debug # pyflakes:ignore - -from django.core.management.base import BaseCommand - -from ietf.stats.utils import find_meetingregistration_person_issues - -class Command(BaseCommand): - help = "Find possible Person/Email objects to repair based on MeetingRegistration objects" - - def add_arguments(self, parser): - parser.add_argument('--meeting',action='append') - - def handle(self, *args, **options): - meetings = options['meeting'] or None - summary = find_meetingregistration_person_issues(meetings) - - print(f'{summary.ok_records} records are OK') - - for msg in summary.could_be_fixed: - print(msg) - - for msg in summary.maybe_address: - print(msg) - - for msg in summary.different_person: - print(msg) - - for msg in summary.no_person: - print(msg) - - for msg in summary.maybe_person: - print(msg) - - for msg in summary.no_email: - print(msg) diff --git a/ietf/stats/management/commands/repair_meetingregistration_person.py b/ietf/stats/management/commands/repair_meetingregistration_person.py deleted file mode 100644 index 74b1a807a6..0000000000 --- a/ietf/stats/management/commands/repair_meetingregistration_person.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright The IETF Trust 2021, All Rights Reserved - -import debug # pyflakes:ignore - -from django.core.management.base import BaseCommand - -from ietf.stats.utils import repair_meetingregistration_person - -class Command(BaseCommand): - help = "Repair MeetingRegistration objects that have no person but an email matching a person" - - def add_arguments(self, parser): - parser.add_argument('--meeting',action='append') - - def handle(self, *args, **options): - meetings = options['meeting'] or None - repaired = repair_meetingregistration_person(meetings) - print(f'Repaired {repaired} MeetingRegistration objects') \ No newline at end of file diff --git a/ietf/stats/tasks.py b/ietf/stats/tasks.py deleted file mode 100644 index 808e797a40..0000000000 --- a/ietf/stats/tasks.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright The IETF Trust 2024, All Rights Reserved -# -# Celery task definitions -# -from celery import shared_task -from django.utils import timezone - -from ietf.meeting.models import Meeting -from ietf.stats.utils import fetch_attendance_from_meetings -from ietf.utils import log - - -@shared_task -def fetch_meeting_attendance_task(): - # fetch most recent two meetings - meetings = Meeting.objects.filter(type="ietf", date__lte=timezone.now()).order_by("-date")[:2] - try: - stats = fetch_attendance_from_meetings(meetings) - except RuntimeError as err: - log.log(f"Error in fetch_meeting_attendance_task: {err}") - else: - for meeting, meeting_stats in zip(meetings, stats): - log.log( - "Fetched data for meeting {:>3}: {:4d} processed, {:4d} added, {:4d} in table".format( - meeting.number, meeting_stats.processed, meeting_stats.added, meeting_stats.total - ) - ) diff --git a/ietf/stats/tests.py b/ietf/stats/tests.py index 47027277be..dc5b5d6ae8 100644 --- a/ietf/stats/tests.py +++ b/ietf/stats/tests.py @@ -1,49 +1,116 @@ -# Copyright The IETF Trust 2016-2020, All Rights Reserved -# -*- coding: utf-8 -*- - +# Copyright The IETF Trust 2016-2026, All Rights Reserved import calendar -import datetime import json +import datetime -from mock import patch +from django.http import Http404 from pyquery import PyQuery -from requests import Response import debug # pyflakes:ignore +from django.test import RequestFactory from django.urls import reverse as urlreverse +from django.utils import timezone +from ietf.meeting.models import Meeting from ietf.utils.test_utils import login_testing_unauthorized, TestCase import ietf.stats.views from ietf.group.factories import RoleFactory -from ietf.meeting.factories import MeetingFactory from ietf.person.factories import PersonFactory from ietf.review.factories import ReviewRequestFactory, ReviewerSettingsFactory, ReviewAssignmentFactory -from ietf.stats.models import MeetingRegistration -from ietf.stats.tasks import fetch_meeting_attendance_task -from ietf.stats.utils import get_meeting_registration_data, FetchStats, fetch_attendance_from_meetings +from ietf.meeting.tests_models import MeetingFactory, RegistrationFactory from ietf.utils.timezone import date_today class StatisticsTests(TestCase): def test_stats_index(self): + # Create a meeting as the index page needs to know the current meeting + MeetingFactory(type_id='ietf', number='124', date=timezone.now()) url = urlreverse(ietf.stats.views.stats_index) r = self.client.get(url) self.assertEqual(r.status_code, 200) def test_document_stats(self): - r = self.client.get(urlreverse("ietf.stats.views.document_stats")) - self.assertRedirects(r, urlreverse("ietf.stats.views.stats_index")) - + # Create a meeting as the index page needs to know the current meeting + MeetingFactory(type_id='ietf', number='124', date=timezone.now()) + r = self.client.get(urlreverse(ietf.stats.views.document_stats)) + self.assertRedirects(r, urlreverse(ietf.stats.views.stats_index)) def test_meeting_stats(self): - r = self.client.get(urlreverse("ietf.stats.views.meeting_stats")) - self.assertRedirects(r, urlreverse("ietf.stats.views.stats_index")) + meeting124 = MeetingFactory(type_id='ietf', number='124', date=timezone.now()) + meeting125 = MeetingFactory(type_id='ietf', number='125', date=timezone.now() + datetime.timedelta(days=120)) + RegistrationFactory.create_batch(15, meeting=meeting124, with_ticket={'attendance_type_id': 'onsite'}, attended=True) + RegistrationFactory(meeting=meeting124, with_ticket={'attendance_type_id': 'onsite'}, attended=False) + RegistrationFactory.create_batch(14, meeting=meeting124, with_ticket={'attendance_type_id': 'remote'}, attended=True) + RegistrationFactory(meeting=meeting124, with_ticket={'attendance_type_id': 'remote'}, attended=False) + RegistrationFactory.create_batch(15, meeting=meeting125, affiliation='Test LLC', with_ticket={'attendance_type_id': 'remote'}, attended=False) + RegistrationFactory.create_batch(25, meeting=meeting125, affiliation='Example, Ltd', with_ticket={'attendance_type_id': 'onsite'}, attended=False) + # Test the meeting specific statitistics per affiliation and per country + r = self.client.get(urlreverse(ietf.stats.views.meeting_stats, kwargs={"meeting_number": "124", "stats_type": "affiliation"})) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "Total Registrations by Affiliation (31 in total)") + self.assertContains(r, "In Person Registrations by Affiliation (16 in total)") + self.assertContains(r, "/stats/meeting/124/affiliation") + self.assertContains(r, "/stats/meeting/125/affiliation") + r = self.client.get(urlreverse(ietf.stats.views.meeting_stats, kwargs={"meeting_number": "124", "stats_type": "country"})) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "Total Registrations by Country (31 in total)") + self.assertContains(r, "In Person Registrations by Country (16 in total)") + self.assertContains(r, "/stats/meeting/124/country") + self.assertContains(r, "/stats/meeting/125/country") + # Test the meetings timeline per country + r = self.client.get(urlreverse(ietf.stats.views.meetings_timeline, kwargs={"stats_type": "country"})) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "/stats/meeting/124/country") + self.assertContains(r, "/stats/meeting/125/country") + self.assertContains(r, "This page provides a timeline of meeting registrations by country") + # Test the meetings timeline per affiliation + r = self.client.get(urlreverse(ietf.stats.views.meetings_timeline, kwargs={"stats_type": "affiliation"})) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "/stats/meeting/124/affiliation") + self.assertContains(r, "/stats/meeting/125/affiliation") + self.assertContains(r, "This page provides a timeline of meeting registrations by affiliation") + # Extract the JSON embedded in the response + pq = PyQuery(r.content) + in_person_data = json.loads(pq.find("script#in-person-chart-data").text()) + self.assertTrue( + any( + ds["label"] == "Example" and ds["data"] == [0, 25] + for ds in in_person_data["datasets"] + ) + ) + # Test the global meetings timeline + r = self.client.get(urlreverse(ietf.stats.views.meetings_timeline, kwargs={"stats_type": "total"})) + self.assertEqual(r.status_code, 200) + self.assertContains(r, "/stats/meeting/124/country") + self.assertContains(r, "/stats/meeting/125/country") + self.assertContains(r, "This page provides a timeline of meeting registrations.") + + def test_meeting_stats_for_bad_meeting(self): + self.assertFalse(Meeting.objects.filter(number=676767).exists()) + for stats_type in ["affiliation", "country"]: + r = self.client.get( + urlreverse( + "ietf.stats.views.meeting_stats", + kwargs={"meeting_number": 676767, "stats_type": stats_type}, + ) + ) + self.assertEqual(r.status_code, 404) + + # We don't have a URL for an interim, but make sure the view will 404 if + # somehow a non-interim gets selected... + interim_num = MeetingFactory(type_id="interim").number + request_factory = RequestFactory() + with self.assertRaises(Http404): + ietf.stats.views.meeting_stats( + request_factory.get(f"/stats/meeting/{interim_num}/{stats_type}"), + meeting_number=interim_num, + stats_type=stats_type, + ) - def test_known_country_list(self): # check redirect url = urlreverse(ietf.stats.views.known_countries_list) @@ -116,120 +183,3 @@ def test_review_stats(self): self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertTrue(q('.review-stats td:contains("1")')) - - @patch('requests.get') - def test_get_meeting_registration_data(self, mock_get): - '''Test function to get reg data. Confirm leading/trailing spaces stripped''' - person = PersonFactory() - data = { - 'LastName': person.last_name() + ' ', - 'FirstName': person.first_name(), - 'Company': 'ABC', - 'Country': 'US', - 'Email': person.email().address, - 'RegType': 'onsite', - 'TicketType': 'week_pass', - 'CheckedIn': 'True', - } - data2 = data.copy() - data2['RegType'] = 'hackathon' - response_a = Response() - response_a.status_code = 200 - response_a._content = json.dumps([data, data2]).encode('utf8') - # second response one less record, it's been deleted - response_b = Response() - response_b.status_code = 200 - response_b._content = json.dumps([data]).encode('utf8') - # mock_get.return_value = response - mock_get.side_effect = [response_a, response_b] - meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016, 7, 14), number="96") - get_meeting_registration_data(meeting) - query = MeetingRegistration.objects.filter( - first_name=person.first_name(), - last_name=person.last_name(), - country_code='US') - self.assertEqual(query.count(), 2) - self.assertEqual(query.filter(reg_type='onsite').count(), 1) - self.assertEqual(query.filter(reg_type='hackathon').count(), 1) - onsite = query.get(reg_type='onsite') - self.assertEqual(onsite.ticket_type, 'week_pass') - self.assertEqual(onsite.checkedin, True) - # call a second time to test delete - get_meeting_registration_data(meeting) - query = MeetingRegistration.objects.filter(meeting=meeting, email=person.email()) - self.assertEqual(query.count(), 1) - self.assertEqual(query.filter(reg_type='onsite').count(), 1) - self.assertEqual(query.filter(reg_type='hackathon').count(), 0) - - @patch('requests.get') - def test_get_meeting_registration_data_duplicates(self, mock_get): - '''Test that get_meeting_registration_data does not create duplicate - MeetingRegistration records - ''' - person = PersonFactory() - data = { - 'LastName': person.last_name() + ' ', - 'FirstName': person.first_name(), - 'Company': 'ABC', - 'Country': 'US', - 'Email': person.email().address, - 'RegType': 'onsite', - 'TicketType': 'week_pass', - 'CheckedIn': 'True', - } - data2 = data.copy() - data2['RegType'] = 'hackathon' - response = Response() - response.status_code = 200 - response._content = json.dumps([data, data2, data]).encode('utf8') - mock_get.return_value = response - meeting = MeetingFactory(type_id='ietf', date=datetime.date(2016, 7, 14), number="96") - self.assertEqual(MeetingRegistration.objects.count(), 0) - get_meeting_registration_data(meeting) - query = MeetingRegistration.objects.all() - self.assertEqual(query.count(), 2) - - @patch("ietf.stats.utils.get_meeting_registration_data") - def test_fetch_attendance_from_meetings(self, mock_get_mtg_reg_data): - mock_meetings = [object(), object(), object()] - mock_get_mtg_reg_data.side_effect = ( - (1, 2, 3), - (4, 5, 6), - (7, 8, 9), - ) - stats = fetch_attendance_from_meetings(mock_meetings) - self.assertEqual( - [mock_get_mtg_reg_data.call_args_list[n][0][0] for n in range(3)], - mock_meetings, - ) - self.assertEqual( - stats, - [ - FetchStats(1, 2, 3), - FetchStats(4, 5, 6), - FetchStats(7, 8, 9), - ] - ) - - -class TaskTests(TestCase): - @patch("ietf.stats.tasks.fetch_attendance_from_meetings") - def test_fetch_meeting_attendance_task(self, mock_fetch_attendance): - today = date_today() - meetings = [ - MeetingFactory(type_id="ietf", date=today - datetime.timedelta(days=1)), - MeetingFactory(type_id="ietf", date=today - datetime.timedelta(days=2)), - MeetingFactory(type_id="ietf", date=today - datetime.timedelta(days=3)), - ] - mock_fetch_attendance.return_value = [FetchStats(1,2,3), FetchStats(1,2,3)] - - fetch_meeting_attendance_task() - self.assertEqual(mock_fetch_attendance.call_count, 1) - self.assertCountEqual(mock_fetch_attendance.call_args[0][0], meetings[0:2]) - - # test handling of RuntimeError - mock_fetch_attendance.reset_mock() - mock_fetch_attendance.side_effect = RuntimeError - fetch_meeting_attendance_task() - self.assertTrue(mock_fetch_attendance.called) - # Good enough that we got here without raising an exception diff --git a/ietf/stats/urls.py b/ietf/stats/urls.py index d2993759d2..01b8758c84 100644 --- a/ietf/stats/urls.py +++ b/ietf/stats/urls.py @@ -11,7 +11,8 @@ url(r"^$", views.stats_index), url(r"^document/(?:(?Pauthors|pages|words|format|formlang|author/(?:documents|affiliation|country|continent|citations|hindex)|yearly/(?:affiliation|country|continent))/)?$", views.document_stats), url(r"^knowncountries/$", views.known_countries_list), - url(r"^meeting/(?P\d+)/(?Pcountry|continent)/$", views.meeting_stats), - url(r"^meeting/(?:(?Poverview|country|continent)/)?$", views.meeting_stats), + url(r"^meeting/$", views.meetings_timeline), + url(r"^meeting/(?P\d+)/(?Paffiliation|country)/$", views.meeting_stats), + url(r"^meeting/(?:(?Paffiliation|country|total)/)?$", views.meetings_timeline), url(r"^review/(?:(?Pcompletion|results|states|time)/)?(?:%(acronym)s/)?$" % settings.URL_REGEXPS, views.review_stats), ] diff --git a/ietf/stats/utils.py b/ietf/stats/utils.py index 2e418eb0eb..a13e87a4f4 100644 --- a/ietf/stats/utils.py +++ b/ietf/stats/utils.py @@ -3,18 +3,12 @@ import re -import requests -from collections import defaultdict, namedtuple - -from django.conf import settings -from django.db.models import Q +from collections import defaultdict import debug # pyflakes:ignore -from ietf.stats.models import AffiliationAlias, AffiliationIgnoredEnding, CountryAlias, MeetingRegistration +from ietf.stats.models import AffiliationAlias, AffiliationIgnoredEnding, CountryAlias from ietf.name.models import CountryName -from ietf.person.models import Person, Email -from ietf.utils.log import log import logging logger = logging.getLogger('django') @@ -221,174 +215,3 @@ def compute_hirsch_index(citation_counts): i += 1 return i - - -def get_meeting_registration_data(meeting): - """"Retrieve registration attendee data and summary statistics. Returns number - of Registration records created. - - MeetingRegistration records are created in realtime as people register for a - meeting. This function serves as an audit / reconciliation. Most records are - expected to already exist. The function has been optimized with this in mind. - """ - num_created = 0 - num_processed = 0 - try: - response = requests.get( - settings.STATS_REGISTRATION_ATTENDEES_JSON_URL.format(number=meeting.number), - timeout=settings.DEFAULT_REQUESTS_TIMEOUT, - ) - except requests.Timeout as exc: - log(f'GET request timed out for [{settings.STATS_REGISTRATION_ATTENDEES_JSON_URL}]: {exc}') - raise RuntimeError("Timeout retrieving data from registrations API") from exc - if response.status_code == 200: - decoded = [] - try: - decoded = response.json() - except ValueError: - if response.content.strip() == 'Invalid meeting': - logger.info('Invalid meeting: {}'.format(meeting.number)) - return (0,0,0) - else: - raise RuntimeError("Could not decode response from registrations API: '%s...'" % (response.content[:64], )) - - records = MeetingRegistration.objects.filter(meeting_id=meeting.pk).select_related('person') - meeting_registrations = {(r.email, r.reg_type):r for r in records} - for registration in decoded: - person = None - # capture the stripped registration values for later use - first_name = registration['FirstName'].strip() - last_name = registration['LastName'].strip() - affiliation = registration['Company'].strip() - country_code = registration['Country'].strip() - address = registration['Email'].strip() - reg_type = registration['RegType'].strip() - ticket_type = registration['TicketType'].strip() - checkedin = bool(registration['CheckedIn'].strip().lower() == 'true') - - if (address, reg_type) in meeting_registrations: - object = meeting_registrations.pop((address, reg_type)) - created = False - else: - object, created = MeetingRegistration.objects.get_or_create( - meeting_id=meeting.pk, - email=address, - reg_type=reg_type) - - if (object.first_name != first_name[:200] or - object.last_name != last_name[:200] or - object.affiliation != affiliation or - object.country_code != country_code or - object.ticket_type != ticket_type or - object.checkedin != checkedin): - object.first_name=first_name[:200] - object.last_name=last_name[:200] - object.affiliation=affiliation - object.country_code=country_code - object.ticket_type=ticket_type - object.checkedin=checkedin - object.save() - - # Add a Person object to MeetingRegistration object - # if valid email is available - if object and not object.person and address: - # If the person already exists do not try to create a new one - emails = Email.objects.filter(address=address) - # there can only be on Email object with a unique email address (primary key) - if emails.exists(): - person = emails.first().person - # Create a new Person object - else: - logger.error("No Person record for registration. email={}".format(address)) - # update the person object to an actual value - object.person = person - object.save() - - if created: - num_created += 1 - num_processed += 1 - - # any registrations left in meeting_registrations no longer exist in reg - # so must have been deleted - for r in meeting_registrations: - try: - MeetingRegistration.objects.get(meeting=meeting,email=r[0],reg_type=r[1]).delete() - logger.info('Removing deleted registration. email={}, reg_type={}'.format(r[0], r[1])) - except MeetingRegistration.DoesNotExist: - pass - else: - raise RuntimeError("Bad response from registrations API: %s, '%s'" % (response.status_code, response.content)) - num_total = MeetingRegistration.objects.filter( - meeting_id=meeting.pk, - reg_type__in=['onsite', 'remote'] - ).filter( - Q(attended=True) | Q(checkedin=True) - ).count() - if meeting.attendees is None or num_total > meeting.attendees: - meeting.attendees = num_total - meeting.save() - return num_created, num_processed, num_total - -def repair_meetingregistration_person(meetings=None): - repaired_records = 0 - qs = MeetingRegistration.objects.all() - if meetings: - qs = qs.filter(meeting__number__in=meetings) - for mr in qs: - if mr.email and not mr.person: - email_person = Person.objects.filter(email__address=mr.email).first() - if email_person: - mr.person = email_person - mr.save() - repaired_records += 1 - return repaired_records - -class MeetingRegistrationIssuesSummary(object): - pass - -def find_meetingregistration_person_issues(meetings=None): - summary = MeetingRegistrationIssuesSummary() - - summary.could_be_fixed = set() - summary.maybe_address = set() - summary.different_person = set() - summary.no_person = set() - summary.maybe_person = set() - summary.no_email = set() - summary.ok_records = 0 - - qs = MeetingRegistration.objects.all() - if meetings: - qs = qs.filter(meeting__number__in=meetings) - for mr in qs: - if mr.person and mr.email and mr.email in mr.person.email_set.values_list('address',flat=True): - summary.ok_records += 1 - elif mr.email: - email_person = Person.objects.filter(email__address=mr.email).first() - if mr.person: - if not email_person: - summary.maybe_address.add(f'{mr.email} is not present in any Email object. The MeetingRegistration object implies this is an address for {mr.person} ({mr.person.pk})') - elif email_person != mr.person: - summary.different_person.add(f'{mr} ({mr.pk}) has person {mr.person} ({mr.person.pk}) but an email {mr.email} attached to a different person {email_person} ({email_person.pk}).') - elif email_person: - summary.could_be_fixed.add(f'{mr} ({mr.pk}) has no person, but email {mr.email} matches {email_person} ({email_person.pk})') - else: - maybe_person_qs = Person.objects.filter(name__icontains=mr.last_name).filter(name__icontains=mr.first_name) - if maybe_person_qs.exists(): - summary.maybe_person.add(f'{mr} ({mr.pk}) has email address {mr.email} which cannot be associated with any Person. Consider these possible people {[(p,p.pk) for p in maybe_person_qs]}') - else: - summary.no_person.add(f'{mr} ({mr.pk}) has email address {mr.email} which cannot be associated with any Person') - else: - summary.no_email.add(f'{mr} ({mr.pk}) provides no email address') - - return summary - - -FetchStats = namedtuple("FetchStats", "added processed total") - - -def fetch_attendance_from_meetings(meetings): - stats = [ - FetchStats(*get_meeting_registration_data(meeting)) for meeting in meetings - ] - return stats diff --git a/ietf/stats/views.py b/ietf/stats/views.py index 504d84e86d..d61c9cab64 100644 --- a/ietf/stats/views.py +++ b/ietf/stats/views.py @@ -9,11 +9,13 @@ import dateutil.relativedelta from collections import defaultdict +from django.conf import settings from django.contrib.auth.decorators import login_required +from django.core.cache import cache from django.http import HttpResponseRedirect -from django.shortcuts import render +from django.shortcuts import render, get_object_or_404 from django.urls import reverse as urlreverse - +from django.db.models import Count import debug # pyflakes:ignore @@ -25,15 +27,32 @@ from ietf.group.models import Role, Group from ietf.person.models import Person from ietf.name.models import ReviewResultName, CountryName, ReviewAssignmentStateName +from ietf.meeting.models import Registration, Meeting from ietf.ietfauth.utils import has_role from ietf.utils.response import permission_denied from ietf.utils.timezone import date_today, DEADLINE_TZINFO +from ietf.meeting.helpers import get_current_ietf_meeting_num +# Color palette for lines +colors = [ + '#FF6384', '#36A2EB', '#FFCE56', '#4BC0C0', '#9966FF', + '#FF9F40', '#C9CBCF', '#7BC043', '#F37735', '#00ABA9', + '#2B5797', '#E81123', '#00A4EF', '#7FBA00', '#FFB900', + '#D83B01', '#B4009E', '#5C2D91', '#008575', '#E3008C', +] def stats_index(request): - return render(request, "stats/index.html") + """Render the statistics index page with the current meeting number as it is required by the meeting menu item.""" + current_meeting = get_current_ietf_meeting_num() + return render(request, "stats/index.html", { + "current_meeting": current_meeting + }) def generate_query_string(query_dict, overrides): + """ + Returns: + A query string starting with '?' if there are parameters, empty string otherwise. + """ query_part = "" if query_dict or overrides: @@ -58,9 +77,20 @@ def generate_query_string(query_dict, overrides): return query_part def get_choice(request, get_parameter, possible_choices, multiple=False): - # the statistics are built with links to make navigation faster, - # so we don't really have a form in most cases, so just use this - # helper instead to select between the choices + """Extract a choice from the request GET parameters. + + Since statistics pages use links for navigation instead of forms, + this helper selects between possible choices from the URL parameters. + + Args: + request: The HTTP request object. + get_parameter: The name of the GET parameter. + possible_choices: List of tuples (value, label). + multiple: If True, return a list of found values; otherwise return the first found or None. + + Returns: + The selected value(s) or None. + """ values = request.GET.getlist(get_parameter) found = [t[0] for t in possible_choices if t[0] in values] @@ -73,75 +103,553 @@ def get_choice(request, get_parameter, possible_choices, multiple=False): return None def add_url_to_choices(choices, url_builder): + """Add URLs to a list of choices. + + Args: + choices: List of tuples (slug, label). + url_builder: Function that takes a slug and returns a URL. + + Returns: + List of tuples (slug, label, url). + """ return [ (slug, label, url_builder(slug)) for slug, label in choices] -def put_into_bin(value, bin_size): - if value is None: - return (0, '') +def document_stats(request, stats_type=None): + # timeline per year, or per specific year: streams, affiliation, rfc vs I-D + # could also be time between individual/WG I-D to rfc publication/IESG ballot + # DISCUSS resolution time + # Humm also split by authors (affiliation) / documents (the rest) probably + """Redirect to the stats index page. Deprecated view.""" + return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) - v = (value // bin_size) * bin_size - return (v, "{} - {}".format(v, v + bin_size - 1)) +def known_countries_list(request, stats_type=None, acronym=None): + """Render a list of known countries with their aliases.""" + countries = CountryName.objects.prefetch_related("countryalias_set") + for c in countries: + # the sorting is a bit of a hack - it puts the ISO code first + # since it was added in a migration + c.aliases = sorted(c.countryalias_set.all(), key=lambda a: a.pk) -def prune_unknown_bin_with_known(bins): - # remove from the unknown bin all authors within the - # named/known bins - all_known = { n for b, names in bins.items() if b for n in names } - bins[""] = [name for name in bins[""] if name not in all_known] - if not bins[""]: - del bins[""] + return render(request, "stats/known_countries_list.html", { + "countries": countries, + }) -def count_bins(bins): - return len({ n for b, names in bins.items() if b for n in names }) +def canonicalize_affiliation(affiliation): + """Canonicalize an affiliation string by removing common suffixes and standardizing prefixes. + + Args: + affiliation: The affiliation string to canonicalize. + + Returns: + The canonicalized affiliation string, or None if input is None. + """ + if not affiliation or affiliation.lower() in ('n/a', 'none', 'unspecified'): + return None + for suffix in ('ab', 'ag', 'corp', 'corp.', 'corporation', 'gmbh', 'inc.', 'inc', 'international pte ltd', 'llc', 'ltd', 'ltd.', 'private limited', 'pty ltd', 'pvt ltd'): + if affiliation.lower().endswith(', ' + suffix): + affiliation = affiliation[:-(len(suffix)+2)] + elif affiliation.lower().endswith(' ' + suffix): + affiliation = affiliation[:-(len(suffix)+1)] + elif affiliation.lower().endswith(',' + suffix): + affiliation = affiliation[:-(len(suffix)+1)] + for prefix in ('akamai','apple', 'cisco', 'futurewei', 'google', 'hitachi', 'hpe', 'huawei', 'juniper', 'meta', 'nokia', 'ntt', 'siemens'): + if affiliation.lower().startswith(prefix + ' '): + affiliation = prefix + return affiliation.title() + +def get_affiliation_data_for_meetings(attendance_type=None): + """Get affiliation participation data for meetings timeline chart. + + Args: + attendance_type: Optional filter for attendance type (e.g., 'onsite'). + + Returns: + Tuple of (sorted_meetings, datasets) for Chart.js. + """ + cache_key = f'stats:get_affiliation_data_for_meetings:{attendance_type}' + sorted_meetings, datasets = cache.get(cache_key, (None, None)) + if (sorted_meetings, datasets) == (None, None): + top_n = 20 # could be a parameter, but would need to adjust cache handling + + # Get registration status details + if attendance_type: + registrations = Registration.objects.filter(tickets__attendance_type=attendance_type) + else: + registrations = Registration.objects.all() + registrations = registrations.values('affiliation', 'meeting__number') + + # Count per canonicalized affiliation + organization = dict() + meetings_set = set() + org_totals = defaultdict(int) + data_map = defaultdict(dict) # {org: {meeting: count}} + + for reg in registrations: + meeting = reg['meeting__number'] + meetings_set.add(meeting) + affiliation = canonicalize_affiliation(reg['affiliation']) or "Unspecified" + organization[affiliation] = organization.get(affiliation, 0) + 1 + org_totals[affiliation] = org_totals.get(affiliation, 0) + 1 + data_map[affiliation][meeting] = data_map[affiliation].get(meeting, 0) + 1 + + # ── Step 2: Sort meetings numerically rather than alphabetically ── + sorted_meetings = sorted(meetings_set, key=lambda x: int(x) if x.isdigit() else x) + + # ── Step 3: Get top N countries ── + top_orgs = sorted( + org_totals.keys(), + key=lambda c: org_totals[c], + reverse=True + )[:top_n] + non_top_orgs = org_totals.keys() - top_orgs + other_totals = defaultdict(int) + for m in sorted_meetings: + other_totals[m] = 0 + for c in non_top_orgs: + other_totals[m] += int(data_map[c].get(m, 0)) + + # ── Step 4: Build Chart.js datasets ── + + datasets = [] + for idx, org in enumerate(top_orgs): + color = colors[idx % len(colors)] + datasets.append({ + 'label': org, + 'data': [data_map[org].get(m, 0) for m in sorted_meetings], + 'borderColor': color, + 'fill': False, + 'tension': 0.3, + 'pointColor': color, + 'pointBackgroundColor': color, + 'pointRadius': 4, + 'pointHoverRadius': 6, + 'borderWidth': 2, + }) + + # -- Step 4.bis handle the other -- + datasets.append({ + 'label': 'Other', + 'data': [other_totals.get(m, 0) for m in sorted_meetings], + 'borderColor': 'black', + 'fill': False, + 'tension': 0.3, + 'pointColor': 'black', + 'pointBackgroundColor': 'black', + 'pointRadius': 4, + 'pointHoverRadius': 6, + 'borderWidth': 2, + }) + cache.set( + cache_key, + (sorted_meetings, datasets), + settings.STATS_TIMELINE_CACHE_TIMEOUT, + ) -def add_labeled_top_series_from_bins(chart_data, bins, limit): - """Take bins on the form (x, label): [name1, name2, ...], figure out - how many there are per label, take the overall top ones and put - them into sorted series like [(x1, len(names1)), (x2, len(names2)), ...].""" - aggregated_bins = defaultdict(set) - xs = set() - for (x, label), names in bins.items(): - xs.add(x) - aggregated_bins[label].update(names) + return sorted_meetings, datasets - xs = list(sorted(xs)) +def get_country_data_for_meetings(attendance_type=None): + """Get country participation data for meetings timeline chart. - sorted_bins = sorted(aggregated_bins.items(), key=lambda t: len(t[1]), reverse=True) - top = [ label for label, names in list(sorted_bins)[:limit]] + Args: + attendance_type: Optional filter for attendance type (e.g., 'onsite'). - for label in top: - series_data = [] + Returns: + Tuple of (sorted_meetings, datasets) for Chart.js. + """ + cache_key = f'stats:get_country_data_for_meetings:{attendance_type}' + sorted_meetings, datasets = cache.get(cache_key, (None, None)) + if (sorted_meetings, datasets) == (None, None): + top_n = 10 # could be a parameter, but would need to adjust cache handling + # Get registration status counts, aggregated by country_code + if attendance_type: + registrations = Registration.objects.filter(tickets__attendance_type=attendance_type) + else: + registrations = Registration.objects.all() + queryset = ( + registrations + .values( + 'meeting__number', # e.g. "118", "119", "120" + 'country_code' # country code of the participant + ) + .annotate(participant_count=Count('id')) + .order_by('meeting__number') # chronological order + ) + + # ── Step 1: Collect all meetings and country totals ── + meetings_set = set() + country_totals = defaultdict(int) + data_map = defaultdict(dict) # {country: {meeting: count}} + + for row in queryset: + meeting = row['meeting__number'] + country = row['country_code'] + count = row['participant_count'] + + meetings_set.add(meeting) + country_totals[country] += count + data_map[country][meeting] = count + + # ── Step 2: Sort meetings numerically rather than alphabetically ── + sorted_meetings = sorted(meetings_set, key=lambda x: int(x) if x.isdigit() else x) + + # ── Step 3: Get top N countries ── + top_countries = sorted( + country_totals.keys(), + key=lambda c: country_totals[c], + reverse=True + )[:top_n] + + # -- Step 3.bis do the 'other' category -- + non_top_countries = country_totals.keys() - top_countries + other_totals = defaultdict(int) + for m in sorted_meetings: + other_totals[m] = 0 + for c in non_top_countries: + other_totals[m] += int(data_map[c].get(m, 0)) + + # ── Step 4: Build Chart.js datasets ── + + datasets = [] + for idx, country in enumerate(top_countries): + color = colors[idx % len(colors)] + datasets.append({ + 'label': country, + 'data': [data_map[country].get(m, 0) for m in sorted_meetings], + 'borderColor': color, + 'fill': False, + 'tension': 0.3, + 'pointColor': color, + 'pointBackgroundColor': color, + 'pointRadius': 4, + 'pointHoverRadius': 6, + 'borderWidth': 2, + }) + + # -- Step 4.bis handle the other -- + datasets.append({ + 'label': 'Other', + 'data': [other_totals.get(m, 0) for m in sorted_meetings], + 'borderColor': 'black', + 'fill': False, + 'tension': 0.3, + 'pointColor': 'black', + 'pointBackgroundColor': 'black', + 'pointRadius': 4, + 'pointHoverRadius': 6, + 'borderWidth': 2, + }) + cache.set( + cache_key, + (sorted_meetings, datasets), + settings.STATS_TIMELINE_CACHE_TIMEOUT, + ) - for x in xs: - names = bins.get((x, label), set()) + return sorted_meetings, datasets + +def get_data_for_meetings(): + """Get total participation data by attendance type for meetings timeline chart. + + Returns: + Tuple of (sorted_meetings, datasets) for Chart.js. + """ + cache_key = "stats:get_data_for_meetings" + sorted_meetings, datasets = cache.get(cache_key, (None, None)) + if (sorted_meetings, datasets) == (None, None): + # Get registration status counts, aggregated by ticket types + registrations = Registration.objects.filter(tickets__attendance_type__in=['onsite', 'remote']) + queryset = ( + registrations + .values( + 'meeting__number', # e.g. "118", "119", "120" + 'tickets__attendance_type' + ) + .annotate(participant_count=Count('id')) + .order_by('meeting__number') # chronological order + ) + + # ── Step 1: Collect all meetings and tickets totals ── + meetings_set = set() + tickets_totals = defaultdict(int) + data_map = defaultdict(dict) # {ticket: {meeting: count}} + + for row in queryset: + meeting = row['meeting__number'] + ticket = row['tickets__attendance_type'] + count = row['participant_count'] + + meetings_set.add(meeting) + tickets_totals[ticket] += count + data_map[ticket][meeting] = count + + # ── Step 2: Sort meetings numerically rather than alphabetically ── + sorted_meetings = sorted(meetings_set, key=lambda x: int(x) if x.isdigit() else x) + ticket_types = tickets_totals.keys() + + # ── Step 4: Build Chart.js datasets ── + # Color palette for lines + colors = [ '#FF6384', '#36A2EB'] + + datasets = [] + for idx, ticket_type in enumerate(ticket_types): + color = colors[idx % len(colors)] + datasets.append({ + 'label': ticket_type, + 'data': [data_map[ticket_type].get(m, 0) for m in sorted_meetings], + 'borderColor': color, + 'backgroundColor': color + '99', # 60% opacity fill + 'fill': True, + 'tension': 0.0, + 'pointColor': color, + 'pointBackgroundColor': color, + 'pointRadius': 4, + 'pointHoverRadius': 6, + 'borderWidth': 2, + }) + cache.set( + cache_key, + (sorted_meetings, datasets), + settings.STATS_TIMELINE_CACHE_TIMEOUT, + ) + return sorted_meetings, datasets + +def meetings_timeline(request, stats_type='country'): + """Render the meetings timeline page with participation statistics over time. + + Args: + request: The HTTP request object. + stats_type: Type of statistics ('country' or 'total'). + top_n: Number of top items to show (for country stats). + + Returns: + Rendered response for the meetings timeline template. + """ + if stats_type == 'total': + total_labels, total_data_sets = get_data_for_meetings() + in_person_labels = ([], []) + in_person_data_sets = ([], []) + top_n = len(total_data_sets) - 1 # subtract one because we don't count "other" + elif stats_type == 'affiliation': + total_labels, total_data_sets = get_affiliation_data_for_meetings() + in_person_labels, in_person_data_sets = get_affiliation_data_for_meetings(attendance_type='onsite') + top_n = len(total_data_sets) - 1 # subtract one because we don't count "other" + elif stats_type == 'country': + total_labels, total_data_sets = get_country_data_for_meetings() + in_person_labels, in_person_data_sets = get_country_data_for_meetings(attendance_type='onsite') + top_n = len(total_data_sets) - 1 # subtract one because we don't count "other" + else: + return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) - series_data.append((x, len(names))) + total_chart_data = { + 'labels': total_labels, + 'datasets': total_data_sets, + } - chart_data.append({ - "data": series_data, - "name": label - }) + # On per country/affiliation have a separate graph for inperson + if stats_type == 'total': + in_person_chart_data = None + else: + in_person_chart_data = { + 'labels': in_person_labels, + 'datasets': in_person_data_sets, + } -def document_stats(request, stats_type=None): - return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) + # Prepare the list of choice buttons for the template + possible_stats_types = [ + ("affiliation", "Per affiliation", urlreverse(meetings_timeline, kwargs={'stats_type': 'affiliation'})), + ("country", "Per country", urlreverse(meetings_timeline, kwargs={'stats_type': 'country'})), + ("total", "Total", urlreverse(meetings_timeline, kwargs={'stats_type': 'total'})), + ] + current_meeting = get_current_ietf_meeting_num() + if stats_type == 'total': + possible_stats_type = 'country' + else: + possible_stats_type = stats_type -def known_countries_list(request, stats_type=None, acronym=None): - countries = CountryName.objects.prefetch_related("countryalias_set") - for c in countries: - # the sorting is a bit of a hack - it puts the ISO code first - # since it was added in a migration - c.aliases = sorted(c.countryalias_set.all(), key=lambda a: a.pk) + possible_meeting_numbers = [(int(current_meeting)-1, urlreverse(meeting_stats, kwargs={'meeting_number': int(current_meeting)-1, 'stats_type': possible_stats_type})), + (int(current_meeting), urlreverse(meeting_stats, kwargs={'meeting_number': int(current_meeting), 'stats_type': possible_stats_type})), + (int(current_meeting)+1, urlreverse(meeting_stats, kwargs={'meeting_number': int(current_meeting)+1, 'stats_type': possible_stats_type}))] - return render(request, "stats/known_countries_list.html", { - "countries": countries, + return render(request, "stats/meetings_timeline.html", { + "top_n": top_n, + "possible_stats_types": possible_stats_types, + "possible_meeting_numbers": possible_meeting_numbers, + "stats_type": stats_type, + "total_chart_data": total_chart_data, + "in_person_chart_data": in_person_chart_data, }) -def meeting_stats(request, num=None, stats_type=None): - return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) +def get_affiliation_data_for_meeting(meeting_number, minimum_required, attendance_type=None): + """Get affiliation participation data for a specific meeting. + + Args: + meeting_number: The meeting number. + minimum_required: Minimum count to include in main data (others go to 'Other'). + attendance_type: Optional filter for attendance type. + + Returns: + Tuple of (labels, data, total) for chart display. + """ + # Get registration status details + registrations = Registration.objects.filter(meeting__number=meeting_number) + if attendance_type: + registrations = registrations.filter(tickets__attendance_type=attendance_type) + registrations = registrations.values('affiliation') + + # Count per canonicalized affiliation + organization = dict() + for reg in registrations: + affiliation = canonicalize_affiliation(reg['affiliation']) or "Unspecified" + organization[affiliation] = organization.get(affiliation, 0) + 1 + + # Sort to have the largest count first (nicer in pie chart) + sorted_orgs = sorted(organization.items(), key=lambda t: t[1], reverse=True) + labels = [] + data = [] + others_count = 0 + total = 0 + for org, count in sorted_orgs: + total += count + if count > minimum_required: + labels.append(org) + data.append(count) + else: + others_count += count + + if others_count > 0: + labels.append('Other') + data.append(others_count) + + return labels, data, total + +def get_data_for_meeting(meeting_number, minimum_required, attendance_type=None): + """Get country participation data for a specific meeting. + + Args: + meeting_number: The meeting number. + minimum_required: Minimum count to include in main data (others go to 'Other'). + attendance_type: Optional filter for attendance type. + + Returns: + Tuple of (labels, data, total) for chart display. + """ + # Get registration status counts, aggregated by country_code + registration_counts = Registration.objects.filter(meeting__number=meeting_number) + if attendance_type: + registration_counts = registration_counts.filter(tickets__attendance_type=attendance_type) + registration_counts = registration_counts.values('country_code').annotate(count=Count('country_code')).order_by('-count') + + labels = [] + data = [] + others_count = 0 + total = 0 + for item in registration_counts: + total += item['count'] + if item['count'] > minimum_required: + labels.append(item['country_code']) + data.append(item['count']) + else: + others_count += item['count'] + + if others_count > 0: + labels.append('Other') + data.append(others_count) + + return labels, data, total + +def meeting_stats(request, meeting_number=None, stats_type='country'): + """Render statistics for a specific meeting. + + Args: + request: The HTTP request object. + meeting_number: The meeting number (defaults to current). + stats_type: Type of statistics ('country' or 'affiliation'). + + Returns: + Rendered response for the meeting stats template. + """ + current_meeting_number = get_current_ietf_meeting_num() + if meeting_number is None: + meeting_number = current_meeting_number + this_meeting = get_object_or_404( + Meeting.objects.filter(type_id="ietf"), number=meeting_number + ) + + if stats_type == 'affiliation': + minimum_required = 4 + total_labels, total_data, total_total = get_affiliation_data_for_meeting(meeting_number, minimum_required) + in_person_labels, in_person_data, in_person_total = get_affiliation_data_for_meeting(meeting_number, minimum_required, attendance_type='onsite') + elif stats_type == 'country': + minimum_required = 10 + total_labels, total_data, total_total = get_data_for_meeting(meeting_number, minimum_required) + in_person_labels, in_person_data, in_person_total = get_data_for_meeting(meeting_number, minimum_required, attendance_type='onsite') + else: + return HttpResponseRedirect(urlreverse("ietf.stats.views.stats_index")) + + total_chart_data = { + 'labels': total_labels, + 'datasets': [{ + 'label': 'Total Registrations by ' + stats_type, + 'data': total_data, + 'borderColor': '#ffffff', + 'borderWidth': 2, + }] + } + in_person_chart_data = { + 'labels': in_person_labels, + 'datasets': [{ + 'label': 'In Person Registrations by ' + stats_type, + 'data': in_person_data, + 'borderColor': '#ffffff', + 'borderWidth': 2, + }] + } + + # Prepare the list of choice buttons for the template + possible_stats_types = [ + ("affiliation", "Per affiliation", urlreverse(meeting_stats, kwargs={'meeting_number': meeting_number, 'stats_type': 'affiliation'})), + ("country", "Per country", urlreverse(meeting_stats, kwargs={'meeting_number': meeting_number, 'stats_type': 'country'})), + ] + + # Prepare the list of meeting number buttons for the template + possible_meeting_numbers = [('All', urlreverse(meetings_timeline, kwargs={'stats_type': stats_type}))] + if int(meeting_number) > 72: # No registration data before IETF-72 + possible_meeting_numbers.append((int(meeting_number)-1, urlreverse(meeting_stats, kwargs={'meeting_number': int(meeting_number)-1, 'stats_type': stats_type}))) + possible_meeting_numbers.append((meeting_number, urlreverse(meeting_stats, kwargs={'meeting_number': meeting_number, 'stats_type': stats_type}))) + if int(meeting_number) <= int(current_meeting_number): # Allow current meeting +1 + possible_meeting_numbers.append((int(meeting_number)+1, urlreverse(meeting_stats, kwargs={'meeting_number': int(meeting_number)+1, 'stats_type': stats_type}))) + + return render(request, "stats/meeting_stats.html", { + "meeting_number": meeting_number, + "meeting_date": this_meeting.date, + "meeting_country": this_meeting.country, + "meeting_city": this_meeting.city, + "possible_stats_types": possible_stats_types, + "possible_meeting_numbers": possible_meeting_numbers, + "stats_type": stats_type, + "minimum_required": minimum_required, + "total_chart_data": total_chart_data, + "total_total": total_total, + "in_person_chart_data": in_person_chart_data, + "in_person_total": in_person_total + }) @login_required def review_stats(request, stats_type=None, acronym=None): + """Render review statistics page with tables and charts for review assignments. + + Shows completion status, results, assignment states, and time series data. + Supports both team-level and reviewer-level views with filtering options. + + Args: + request: The HTTP request object. + stats_type: Type of statistics ('completion', 'results', 'states', 'time'). + acronym: Team acronym for reviewer-level view (None for team view). + + Returns: + Rendered response for the review stats template. + """ # This view is a bit complex because we want to show a bunch of # tables with various filtering options, and both a team overview # and a reviewers-within-team overview - and a time series chart. diff --git a/ietf/submit/checkers.py b/ietf/submit/checkers.py index 89908748a7..e02b686576 100644 --- a/ietf/submit/checkers.py +++ b/ietf/submit/checkers.py @@ -18,7 +18,7 @@ from ietf.utils import tool_version from ietf.utils.log import log, assertion from ietf.utils.pipe import pipe -from ietf.utils.test_runner import set_coverage_checking +from ietf.utils.test_runner import disable_coverage class DraftSubmissionChecker(object): name = "" @@ -247,34 +247,33 @@ def check_file_txt(self, path): ) # yanglint - set_coverage_checking(False) # we can't count the following as it may or may not be run, depending on setup - if settings.SUBMIT_YANGLINT_COMMAND and os.path.exists(settings.YANGLINT_BINARY): - cmd_template = settings.SUBMIT_YANGLINT_COMMAND - command = [ w for w in cmd_template.split() if not '=' in w ][0] - cmd = cmd_template.format(model=path, rfclib=settings.SUBMIT_YANG_RFC_MODEL_DIR, tmplib=workdir, - draftlib=settings.SUBMIT_YANG_DRAFT_MODEL_DIR, ianalib=settings.SUBMIT_YANG_IANA_MODEL_DIR, - cataloglib=settings.SUBMIT_YANG_CATALOG_MODEL_DIR, ) - code, out, err = pipe(cmd) - out = out.decode('utf-8') - err = err.decode('utf-8') - if code > 0 or len(err.strip()) > 0: - err_lines = err.splitlines() - for line in err_lines: - if line.strip(): - try: - if 'err : ' in line: - errors += 1 - if 'warn: ' in line: - warnings += 1 - except ValueError: - pass - #passed = passed and code == 0 # For the submission tool. Yang checks always pass - message += "{version}: {template}:\n{output}\n".format( - version=tool_version[command], - template=cmd_template, - output=out + "No validation errors\n" if (code == 0 and len(err) == 0) else out + err, - ) - set_coverage_checking(True) + with disable_coverage(): # pragma: no cover + if settings.SUBMIT_YANGLINT_COMMAND and os.path.exists(settings.YANGLINT_BINARY): + cmd_template = settings.SUBMIT_YANGLINT_COMMAND + command = [ w for w in cmd_template.split() if not '=' in w ][0] + cmd = cmd_template.format(model=path, rfclib=settings.SUBMIT_YANG_RFC_MODEL_DIR, tmplib=workdir, + draftlib=settings.SUBMIT_YANG_DRAFT_MODEL_DIR, ianalib=settings.SUBMIT_YANG_IANA_MODEL_DIR, + cataloglib=settings.SUBMIT_YANG_CATALOG_MODEL_DIR, ) + code, out, err = pipe(cmd) + out = out.decode('utf-8') + err = err.decode('utf-8') + if code > 0 or len(err.strip()) > 0: + err_lines = err.splitlines() + for line in err_lines: + if line.strip(): + try: + if 'err : ' in line: + errors += 1 + if 'warn: ' in line: + warnings += 1 + except ValueError: + pass + #passed = passed and code == 0 # For the submission tool. Yang checks always pass + message += "{version}: {template}:\n{output}\n".format( + version=tool_version[command], + template=cmd_template, + output=out + "No validation errors\n" if (code == 0 and len(err) == 0) else out + err, + ) else: errors += 1 message += "No such file: %s\nPossible mismatch between extracted xym file name and returned module name?\n" % (path) diff --git a/ietf/submit/migrations/0002_alter_submission_xml_version.py b/ietf/submit/migrations/0002_alter_submission_xml_version.py new file mode 100644 index 0000000000..275e6efd95 --- /dev/null +++ b/ietf/submit/migrations/0002_alter_submission_xml_version.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.23 on 2025-08-01 19:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("submit", "0001_initial"), + ] + + operations = [ + migrations.AlterField( + model_name="submission", + name="xml_version", + field=models.CharField(blank=True, default=None, max_length=4, null=True), + ), + ] diff --git a/ietf/submit/migrations/0003_alter_submission_authors_alter_submissioncheck_items.py b/ietf/submit/migrations/0003_alter_submission_authors_alter_submissioncheck_items.py new file mode 100644 index 0000000000..2c51659204 --- /dev/null +++ b/ietf/submit/migrations/0003_alter_submission_authors_alter_submissioncheck_items.py @@ -0,0 +1,46 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations, models + + +def forward(apps, schema_editor): + JSON_ENCODED_NULL = r"\u0000" + NULL = "\x00" + NUL_SYMBOL = "\u2400" # Unicode single-char "NUL" symbol + + Submission = apps.get_model("submit", "Submission") + # The qs filter sees the serialized JSON string... + null_in_authors = Submission.objects.filter(authors__contains=JSON_ENCODED_NULL) + for submission in null_in_authors: + # submission.authors is now deserialized into Python objects + for author in submission.authors: + for k in author: + author[k] = author[k].replace(NULL, NUL_SYMBOL) + submission.save() + + +def reverse(apps, schema_editor): + pass # don't restore invalid data + + +class Migration(migrations.Migration): + dependencies = [ + ("submit", "0002_alter_submission_xml_version"), + ] + + operations = [ + migrations.RunPython(forward, reverse), + migrations.AlterField( + model_name="submission", + name="authors", + field=models.JSONField( + default=list, + help_text="List of authors with name, email, affiliation and country.", + ), + ), + migrations.AlterField( + model_name="submissioncheck", + name="items", + field=models.JSONField(blank=True, default=dict, null=True), + ), + ] diff --git a/ietf/submit/models.py b/ietf/submit/models.py index 51f7541e31..576ba3e114 100644 --- a/ietf/submit/models.py +++ b/ietf/submit/models.py @@ -3,7 +3,6 @@ import email -import jsonfield from django.db import models from django.utils import timezone @@ -46,7 +45,9 @@ class Submission(models.Model): words = models.IntegerField(null=True, blank=True) formal_languages = models.ManyToManyField(FormalLanguageName, blank=True, help_text="Formal languages used in document") - authors = jsonfield.JSONField(default=list, help_text="List of authors with name, email, affiliation and country.") + authors = models.JSONField(default=list, help_text="List of authors with name, email, affiliation and country.") + # Schema note: authors is a list of authors. Each author is a JSON object with + # "name", "email", "affiliation", and "country" keys. All values are strings. note = models.TextField(blank=True) replaces = models.CharField(max_length=1000, blank=True) @@ -55,7 +56,7 @@ class Submission(models.Model): file_size = models.IntegerField(null=True, blank=True) document_date = models.DateField(null=True, blank=True) submission_date = models.DateField(default=date_today) - xml_version = models.CharField(null=True, max_length=4, default=None) + xml_version = models.CharField(null=True, blank=True, max_length=4, default=None) submitter = models.CharField(max_length=255, blank=True, help_text="Name and email of submitter, e.g. \"John Doe <john@example.org>\".") @@ -135,7 +136,7 @@ class SubmissionCheck(models.Model): message = models.TextField(null=True, blank=True) errors = models.IntegerField(null=True, blank=True, default=None) warnings = models.IntegerField(null=True, blank=True, default=None) - items = jsonfield.JSONField(null=True, blank=True, default='{}') + items = models.JSONField(null=True, blank=True, default=dict) symbol = models.CharField(max_length=64, default='') # def __str__(self): diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index 6b1c998e76..ad361d31b2 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -1,11 +1,11 @@ -# Copyright The IETF Trust 2011-2023, All Rights Reserved +# Copyright The IETF Trust 2011-2026, All Rights Reserved # -*- coding: utf-8 -*- import datetime import email import io -import mock +from unittest import mock import os import re import sys @@ -51,8 +51,9 @@ process_submission_xml, process_uploaded_submission, process_and_validate_submission, apply_yang_checker_to_draft, run_all_yang_model_checks) +from ietf.submit.views import access_token_is_valid, auth_token_is_valid from ietf.utils import tool_version -from ietf.utils.accesstoken import generate_access_token +from ietf.utils.accesstoken import generate_access_token, generate_random_key from ietf.utils.mail import outbox, get_payload_text from ietf.utils.test_runner import TestBlobstoreManager from ietf.utils.test_utils import login_testing_unauthorized, TestCase @@ -206,20 +207,24 @@ def test_manualpost_view(self): r = self.client.get(url) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertIn( - urlreverse( - "ietf.submit.views.submission_status", - kwargs=dict(submission_id=submission.pk) - ), - q("#manual.submissions td a").attr("href") - ) - self.assertIn( - submission.name, - q("#manual.submissions td a").text() + # Validate that the basic submission status URL is on the manual post page + # _without_ an access token, even if logged in as various users. + expected_url = urlreverse( + "ietf.submit.views.submission_status", + kwargs=dict(submission_id=submission.pk) ) + selected_elts = q("#manual.submissions td a") + self.assertEqual(expected_url, selected_elts.attr("href")) + self.assertIn(submission.name, selected_elts.text()) + for username in ["plain", "secretary"]: + self.client.login(username=username, password=username + "+password") + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + selected_elts = q("#manual.submissions td a") + self.assertEqual(expected_url, selected_elts.attr("href")) + self.assertIn(submission.name, selected_elts.text()) - def test_manualpost_cancel(self): - pass class SubmitTests(BaseSubmitTestCase): def setUp(self): @@ -594,7 +599,7 @@ def submit_existing(self, formats, change_authors=True, group_type='wg', stream_ TestBlobstoreManager().emptyTestBlobstores() def _assert_authors_are_action_holders(draft, expect=True): - for author in draft.authors(): + for author in draft.author_persons(): if expect: self.assertIn(author, draft.action_holders.all()) else: @@ -2403,7 +2408,7 @@ def test_upload_draft(self): response = r.json() self.assertCountEqual( response.keys(), - ['id', 'name', 'rev', 'status_url'], + ['id', 'name', 'rev', 'status_url', 'submission_url'], ) submission_id = int(response['id']) self.assertEqual(response['name'], 'draft-somebody-test') @@ -2415,6 +2420,13 @@ def test_upload_draft(self): kwargs={'submission_id': submission_id}, ), ) + self.assertEqual( + response['submission_url'], + 'https://datatracker.example.com' + urlreverse( + 'ietf.submit.views.submission_status', + kwargs={'submission_id': submission_id}, + ) + ) self.assertEqual(mock_task.delay.call_count, 1) self.assertEqual(mock_task.delay.call_args.args, (submission_id,)) submission = Submission.objects.get(pk=submission_id) @@ -2981,7 +2993,7 @@ def test_process_and_accept_uploaded_submission_invalid(self): xml_path = Path(settings.IDSUBMIT_STAGING_PATH) / 'draft-somebody-test-00.xml' with xml_path.open('w') as f: f.write(xml_data) - store_str("staging", "draft-somebody-test-00.xml", xml_data) + store_str("staging", "draft-somebody-test-00.xml", xml_data, allow_overwrite=True) with mock.patch( 'ietf.submit.utils.apply_checkers', side_effect = lambda _, __: submission.checks.create( @@ -3047,25 +3059,25 @@ def test_process_submission_xml(self): # Should behave on missing or partial elements TestBlobstoreManager().emptyTestBlobstores() xml_path.write_text(re.sub(r"", "", xml_contents)) # strip entirely - store_str("staging", "draft-somebody-test-00.xml", re.sub(r"", "", xml_contents)) + store_str("staging", "draft-somebody-test-00.xml", re.sub(r"", "", xml_contents), allow_overwrite=True) output = process_submission_xml("draft-somebody-test", "00") self.assertEqual(output["document_date"], None) TestBlobstoreManager().emptyTestBlobstores() xml_path.write_text(re.sub(r")", r"\1 day=\2", xml_contents)) # remove month - store_str("staging", "draft-somebody-test-00.xml", re.sub(r"()", r"\1 day=\2", xml_contents)) + store_str("staging", "draft-somebody-test-00.xml", re.sub(r"()", r"\1 day=\2", xml_contents), allow_overwrite=True) output = process_submission_xml("draft-somebody-test", "00") self.assertEqual(output["document_date"], date_today()) TestBlobstoreManager().emptyTestBlobstores() xml_path.write_text(re.sub(r"", r"", xml_contents)) # remove day - store_str("staging", "draft-somebody-test-00.xml", re.sub(r"", r"", xml_contents)) + store_str("staging", "draft-somebody-test-00.xml", re.sub(r"", r"", xml_contents), allow_overwrite=True) output = process_submission_xml("draft-somebody-test", "00") self.assertEqual(output["document_date"], date_today()) @@ -3080,7 +3092,7 @@ def test_process_submission_xml(self): ) xml_path.write_text(xml.read()) xml.seek(0) - store_str("staging", "draft-somebody-test-00.xml", xml.read()) + store_str("staging", "draft-somebody-test-00.xml", xml.read(), allow_overwrite=True) with self.assertRaisesMessage(SubmissionError, "disagrees with submission filename"): process_submission_xml("draft-somebody-test", "00") @@ -3095,7 +3107,7 @@ def test_process_submission_xml(self): ) xml_path.write_text(xml.read()) xml.seek(0) - store_str("staging", "draft-somebody-test-00.xml", xml.read()) + store_str("staging", "draft-somebody-test-00.xml", xml.read(), allow_overwrite=True) with self.assertRaisesMessage(SubmissionError, "disagrees with submission revision"): process_submission_xml("draft-somebody-test", "00") @@ -3110,7 +3122,7 @@ def test_process_submission_xml(self): ) xml_path.write_text(xml.read()) xml.seek(0) - store_str("staging", "draft-somebody-test-00.xml", xml.read()) + store_str("staging", "draft-somebody-test-00.xml", xml.read(), allow_overwrite=True) with self.assertRaisesMessage(SubmissionError, "Could not extract a valid title"): process_submission_xml("draft-somebody-test", "00") @@ -3153,7 +3165,7 @@ def test_process_submission_text(self): with txt_path.open('w') as fd: fd.write(txt.read()) txt.seek(0) - store_str("staging", "draft-somebody-test-00.txt", txt.read()) + store_str("staging", "draft-somebody-test-00.txt", txt.read(), allow_overwrite=True) txt.close() with self.assertRaisesMessage(SubmissionError, 'disagrees with submission filename'): process_submission_text("draft-somebody-test", "00") @@ -3170,7 +3182,7 @@ def test_process_submission_text(self): with txt_path.open('w') as fd: fd.write(txt.read()) txt.seek(0) - store_str("staging", "draft-somebody-test-00.txt", txt.read()) + store_str("staging", "draft-somebody-test-00.txt", txt.read(), allow_overwrite=True) txt.close() with self.assertRaisesMessage(SubmissionError, 'disagrees with submission revision'): process_submission_text("draft-somebody-test", "00") @@ -3500,3 +3512,31 @@ def test_submissionerror(self, mock_sanitize_message): mock_sanitize_message.call_args_list, [mock.call("hi"), mock.call("there")], ) + + +class HelperTests(TestCase): + def test_access_token_is_valid(self): + submission: Submission = SubmissionFactory() # type: ignore + valid_token = submission.access_token() + access_key = submission.access_key # accept this for backwards compat + invalid_token = "not the valid token" + self.assertTrue(access_token_is_valid(submission, valid_token)) + self.assertTrue(access_token_is_valid(submission, access_key)) + self.assertFalse(access_token_is_valid(submission, invalid_token)) + + def test_auth_token_is_valid(self): + auth_key = generate_random_key() + submission: Submission = SubmissionFactory(auth_key = auth_key) # type: ignore + valid_token = generate_access_token(submission.auth_key) + auth_key = submission.auth_key # accept this for backwards compat + invalid_token = "not the valid token" + self.assertTrue(auth_token_is_valid(submission, valid_token)) + self.assertTrue(auth_token_is_valid(submission, auth_key)) + self.assertFalse(auth_token_is_valid(submission, invalid_token)) + + submission.auth_key = "" + submission.save() + self.assertFalse(auth_token_is_valid(submission, valid_token)) + self.assertFalse(auth_token_is_valid(submission, auth_key)) + self.assertFalse(auth_token_is_valid(submission, invalid_token)) + self.assertFalse(auth_token_is_valid(submission, "")) diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py index a0c7dd8511..457462e4f2 100644 --- a/ietf/submit/utils.py +++ b/ietf/submit/utils.py @@ -395,10 +395,7 @@ def post_submission(request, submission, approved_doc_desc, approved_subm_desc): log.log(f"{submission.name}: updated state and info") - trouble = rebuild_reference_relations(draft, find_submission_filenames(draft)) - if trouble: - log.log('Rebuild_reference_relations trouble: %s'%trouble) - log.log(f"{submission.name}: rebuilt reference relations") + rebuild_reference_relations(draft, find_submission_filenames(draft)) if draft.stream_id == "ietf" and draft.group.type_id == "wg" and draft.rev == "00": # automatically set state "WG Document" @@ -1268,7 +1265,7 @@ def process_submission_text(filename, revision): if title: title = _normalize_title(title) - # Translation taable drops \r, \n, <, >. + # Translation table drops \r, \n, <, >. trans_table = str.maketrans("", "", "\r\n<>") authors = [ { @@ -1615,6 +1612,4 @@ def active(dirent): log.log(f"Error processing {item.name}: {e}") ftp_moddir = Path(settings.FTP_DIR) / "yang" / "draftmod/" - if not moddir.endswith("/"): - moddir += "/" - subprocess.call(("/usr/bin/rsync", "-aq", "--delete", moddir, ftp_moddir)) + subprocess.call(("/usr/bin/rsync", "-aq", "--delete", f"{moddir}/", str(ftp_moddir))) diff --git a/ietf/submit/views.py b/ietf/submit/views.py index 2b9b55c00e..2db3f51098 100644 --- a/ietf/submit/views.py +++ b/ietf/submit/views.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- import re import datetime +from secrets import compare_digest from typing import Optional, cast # pyflakes:ignore from urllib.parse import urljoin @@ -181,6 +182,10 @@ def err(code, error, messages=None): settings.IDTRACKER_BASE_URL, urlreverse(api_submission_status, kwargs={'submission_id': submission.pk}), ), + 'submission_url': urljoin( + settings.IDTRACKER_BASE_URL, + urlreverse("ietf.submit.views.submission_status", kwargs={'submission_id': submission.pk}), + ), } ) else: @@ -255,19 +260,48 @@ def search_submission(request): ) -def can_edit_submission(user, submission, access_token): - key_matched = access_token and submission.access_token() == access_token - if not key_matched: key_matched = submission.access_key == access_token # backwards-compat - return key_matched or has_role(user, "Secretariat") +def access_token_is_valid(submission: Submission, access_token: str): + """Check whether access_token is valid for submission, in constant time""" + token_matched = compare_digest(submission.access_token(), access_token) + # also compare key directly for backwards compatibility + key_matched = compare_digest(submission.access_key, access_token) + return token_matched or key_matched + + +def auth_token_is_valid(submission: Submission, auth_token: str): + """Check whether auth_token is valid for submission, in constant time""" + auth_key = submission.auth_key + if not auth_key: + # Make the same calls as the other branch to keep constant time, then + # return False because there is no auth key + compare_digest(generate_access_token("fake"), auth_token) + compare_digest("fake", auth_token) + return False + else: + token_matched = compare_digest(generate_access_token(auth_key), auth_token) + # also compare key directly for backwards compatibility + key_matched = compare_digest(auth_key, auth_token) + return token_matched or key_matched + + +def can_edit_submission(user, submission: Submission, access_token: str | None): + if has_role(user, "Secretariat"): + return True + elif not access_token: + return False + return access_token_is_valid(submission, access_token) + def submission_status(request, submission_id, access_token=None): # type: (HttpRequest, str, Optional[str]) -> HttpResponse submission = get_object_or_404(Submission, pk=submission_id) - key_matched = access_token and submission.access_token() == access_token - if not key_matched: key_matched = submission.access_key == access_token # backwards-compat - if access_token and not key_matched: - raise Http404 + if access_token: + key_matched = access_token_is_valid(submission, access_token) + if not key_matched: + raise Http404 + else: + key_matched = False if submission.state.slug == "cancel": errors = {} @@ -621,8 +655,7 @@ def edit_submission(request, submission_id, access_token=None): def confirm_submission(request, submission_id, auth_token): submission = get_object_or_404(Submission, pk=submission_id) - key_matched = submission.auth_key and auth_token == generate_access_token(submission.auth_key) - if not key_matched: key_matched = auth_token == submission.auth_key # backwards-compat + key_matched = submission.auth_key and auth_token_is_valid(submission, auth_token) if request.method == 'POST' and submission.state_id in ("auth", "aut-appr") and key_matched: # Set a temporary state 'confirmed' to avoid entering this code @@ -755,4 +788,7 @@ def get_submission_or_404(submission_id, access_token=None): def async_poke_test(request): result = poke.delay() - return HttpResponse(f'Poked {result}', content_type='text/plain') + return HttpResponse( + f'Poked {result}', + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + ) diff --git a/ietf/sync/errata.py b/ietf/sync/errata.py new file mode 100644 index 0000000000..113d987291 --- /dev/null +++ b/ietf/sync/errata.py @@ -0,0 +1,184 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +import datetime +import json +from collections import defaultdict +from typing import DefaultDict + +from django.conf import settings +from django.core.files.storage import storages +from django.db import transaction +from django.db.models import Q + +from ietf.doc.models import Document, DocEvent +from ietf.name.models import DocTagName +from ietf.person.models import Person +from ietf.utils.log import log +from ietf.utils.models import DirtyBits + + +DEFAULT_ERRATA_JSON_BLOB_NAME = "other/errata.json" + +type ErrataJsonEntry = dict[str, str] + +def get_errata_last_updated() -> datetime.datetime: + """Get timestamp of the last errata.json update + + May raise FileNotFoundError or other storage/S3 exceptions. Be prepared. + """ + red_bucket = storages["red_bucket"] + return red_bucket.get_modified_time( + getattr(settings, "ERRATA_JSON_BLOB_NAME", DEFAULT_ERRATA_JSON_BLOB_NAME) + ) + + +def get_errata_data() -> list[ErrataJsonEntry]: + red_bucket = storages["red_bucket"] + with red_bucket.open( + getattr(settings, "ERRATA_JSON_BLOB_NAME", DEFAULT_ERRATA_JSON_BLOB_NAME), "r" + ) as f: + errata_data = json.load(f) + return errata_data + + +def errata_map_from_json(errata_data: list[ErrataJsonEntry]): + """Create a dict mapping RFC number to a list of applicable errata records""" + errata = defaultdict(list) + for item in errata_data: + doc_id = item["doc-id"] + if doc_id.upper().startswith("RFC"): + rfc_number = int(doc_id[3:]) + errata[rfc_number].append(item) + return dict(errata) + + +def update_errata_tags(errata_data: list[ErrataJsonEntry]): + tag_has_errata = DocTagName.objects.get(slug="errata") + tag_has_verified_errata = DocTagName.objects.get(slug="verified-errata") + system = Person.objects.get(name="(System)") + + errata_map = errata_map_from_json(errata_data) + nums_with_errata = [ + num + for num, errata in errata_map.items() + if any(er["errata_status_code"] != "Rejected" for er in errata) + ] + nums_with_verified_errata = [ + num + for num, errata in errata_map.items() + if any(er["errata_status_code"] == "Verified" for er in errata) + ] + + rfcs_gaining_errata_tag = Document.objects.filter( + type_id="rfc", rfc_number__in=nums_with_errata + ).exclude(tags=tag_has_errata) + + rfcs_gaining_verified_errata_tag = Document.objects.filter( + type_id="rfc", rfc_number__in=nums_with_verified_errata + ).exclude(tags=tag_has_verified_errata) + + rfcs_losing_errata_tag = Document.objects.filter( + type_id="rfc", tags=tag_has_errata + ).exclude(rfc_number__in=nums_with_errata) + + rfcs_losing_verified_errata_tag = Document.objects.filter( + type_id="rfc", tags=tag_has_verified_errata + ).exclude(rfc_number__in=nums_with_verified_errata) + + # map rfc_number to add/remove lists + changes: DefaultDict[Document, dict[str, list[DocTagName]]] = defaultdict( + lambda: {"add": [], "remove": []} + ) + for rfc in rfcs_gaining_errata_tag: + changes[rfc]["add"].append(tag_has_errata) + for rfc in rfcs_gaining_verified_errata_tag: + changes[rfc]["add"].append(tag_has_verified_errata) + for rfc in rfcs_losing_errata_tag: + changes[rfc]["remove"].append(tag_has_errata) + for rfc in rfcs_losing_verified_errata_tag: + changes[rfc]["remove"].append(tag_has_verified_errata) + + for rfc, changeset in changes.items(): + # Update in a transaction per RFC to keep tags and DocEvents consistent. + # With this in place, an interrupted task will be cleanly completed on the + # next run. + with transaction.atomic(): + change_descs = [] + for tag in changeset["add"]: + rfc.tags.add(tag) + change_descs.append(f"added {tag.slug} tag") + for tag in changeset["remove"]: + rfc.tags.remove(tag) + change_descs.append(f"removed {tag.slug} tag") + summary = "Update from RFC Editor: " + ", ".join(change_descs) + if rfc.rfc_number in errata_map and all( + er["errata_status_code"] == "Rejected" + for er in errata_map[rfc.rfc_number] + ): + summary += " (all errata rejected)" + DocEvent.objects.create( + doc=rfc, + rev=rfc.rev, # expect no rev + by=system, + type="sync_from_rfc_editor", + desc=summary, + ) + + +def update_errata_from_rfceditor(): + errata_data = get_errata_data() + update_errata_tags(errata_data) + + +## DirtyBits management for the errata tags + +ERRATA_SLUG = DirtyBits.Slugs.ERRATA + + +def update_errata_dirty_time() -> DirtyBits | None: + try: + last_update = get_errata_last_updated() + except Exception as err: + log(f"Error in get_errata_last_updated: {err}") + return None + else: + dirty_work, created = DirtyBits.objects.update_or_create( + slug=ERRATA_SLUG, defaults={"dirty_time": last_update} + ) + if created: + log(f"Created DirtyBits(slug='{ERRATA_SLUG}')") + return dirty_work + + +def mark_errata_as_processed(when: datetime.datetime): + n_updated = DirtyBits.objects.filter( + Q(processed_time__isnull=True) | Q(processed_time__lt=when), + slug=ERRATA_SLUG, + ).update(processed_time=when) + if n_updated > 0: + log(f"processed_time is now {when.isoformat()}") + else: + log("processed_time not updated, no matching record found") + + +def errata_are_dirty(): + """Does the rfc index need to be updated?""" + dirty_work = update_errata_dirty_time() # creates DirtyBits if needed + if dirty_work is None: + # A None indicates we could not check the timestamp of errata.json. In that + # case, we are not likely to be able to read the blob either, so don't try + # to process it. An error was already logged. + return False + display_processed_time = ( + dirty_work.processed_time.isoformat() + if dirty_work.processed_time is not None + else "never" + ) + log( + f"DirtyBits(slug='{ERRATA_SLUG}'): " + f"dirty_time={dirty_work.dirty_time.isoformat()} " + f"processed_time={display_processed_time}" + ) + return ( + dirty_work.processed_time is None + or dirty_work.dirty_time >= dirty_work.processed_time + ) diff --git a/ietf/sync/iana.py b/ietf/sync/iana.py index f46fe407d4..0d40c5337e 100644 --- a/ietf/sync/iana.py +++ b/ietf/sync/iana.py @@ -66,8 +66,8 @@ def update_rfc_log_from_protocol_page(rfc_names, rfc_must_published_later_than): def fetch_changes_json(url, start, end): - url += "?start=%s&end=%s" % (urlquote(start.astimezone(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S")), - urlquote(end.astimezone(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S"))) + url += "?start=%s&end=%s" % (urlquote(start.astimezone(datetime.UTC).strftime("%Y-%m-%d %H:%M:%S")), + urlquote(end.astimezone(datetime.UTC).strftime("%Y-%m-%d %H:%M:%S"))) # HTTP basic auth username = "ietfsync" password = settings.IANA_SYNC_PASSWORD @@ -161,7 +161,7 @@ def update_history_with_changes(changes, send_email=True): for c in changes: docname = c['doc'] - timestamp = datetime.datetime.strptime(c["time"], "%Y-%m-%d %H:%M:%S",).replace(tzinfo=datetime.timezone.utc) + timestamp = datetime.datetime.strptime(c["time"], "%Y-%m-%d %H:%M:%S",).replace(tzinfo=datetime.UTC) if c['type'] in ("iana_state", "iana_review"): if c['type'] == "iana_state": @@ -247,7 +247,7 @@ def parse_review_email(text): review_time = parsedate_to_datetime(msg["Date"]) # parsedate_to_datetime() may return a naive timezone - treat as UTC if review_time.tzinfo is None or review_time.tzinfo.utcoffset(review_time) is None: - review_time = review_time.replace(tzinfo=datetime.timezone.utc) + review_time = review_time.replace(tzinfo=datetime.UTC) # by by = None diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py index 889317cdcf..347b58efbb 100644 --- a/ietf/sync/rfceditor.py +++ b/ietf/sync/rfceditor.py @@ -1,20 +1,16 @@ -# Copyright The IETF Trust 2012-2020, All Rights Reserved +# Copyright The IETF Trust 2012-2025, All Rights Reserved # -*- coding: utf-8 -*- -import base64 import datetime import re -import requests from typing import Iterator, Optional, Union -from urllib.parse import urlencode from xml.dom import pulldom, Node -from django.conf import settings +from django.db import transaction from django.db.models import Subquery, OuterRef, F, Q from django.utils import timezone -from django.utils.encoding import smart_bytes, force_str import debug # pyflakes:ignore @@ -30,9 +26,9 @@ from ietf.utils.mail import send_mail_text from ietf.utils.timezone import datetime_from_date, RPC_TZINFO -#QUEUE_URL = "https://www.rfc-editor.org/queue2.xml" -#INDEX_URL = "https://www.rfc-editor.org/rfc/rfc-index.xml" -#POST_APPROVED_DRAFT_URL = "https://www.rfc-editor.org/sdev/jsonexp/jsonparser.php" +# QUEUE_URL = "https://www.rfc-editor.org/queue2.xml" +# INDEX_URL = "https://www.rfc-editor.org/rfc/rfc-index.xml" +# POST_APPROVED_DRAFT_URL = "https://www.rfc-editor.org/sdev/jsonexp/jsonparser.php" MIN_ERRATA_RESULTS = 5000 MIN_INDEX_RESULTS = 8000 @@ -114,6 +110,8 @@ def parse_queue(response): stream = "irtf" elif name.startswith("INDEPENDENT"): stream = "ise" + elif name.startswith("Editorial Stream"): + stream = "editorial" else: stream = None warnings.append("unrecognized section " + name) @@ -427,7 +425,7 @@ def update_docs_from_rfc_index( pass # Logging below warning turns out to be unhelpful - there are many references # to such things in the index: - # * all april-1 RFCs have an internal name that looks like a draft name, but there + # * all april-1 RFCs have an internal name that looks like a draft name, but there # was never such a draft. More of these will exist in the future # * Several documents were created with out-of-band input to the RFC-editor, for a # variety of reasons. @@ -436,7 +434,7 @@ def update_docs_from_rfc_index( # If there is no draft to point to, don't point to one, even if there was an RPC # internal name in use (and in the RPC database). This will be a requirement on the # reimplementation of the creation of the rfc-index. - # + # # log(f"Warning: RFC index for {rfc_number} referred to unknown draft {draft_name}") # Find or create the RFC document @@ -465,14 +463,18 @@ def update_docs_from_rfc_index( doc.set_state(rfc_published_state) if draft: doc.formal_languages.set(draft.formal_languages.all()) - for author in draft.documentauthor_set.all(): - # Copy the author but point at the new doc. + # Create authors based on the last draft in the datatracker. This + # path will go away when we publish via the modernized RPC workflow + # but until then, these are the only data we have for authors that + # are easily connected to Person records. + for documentauthor in draft.documentauthor_set.all(): + # Copy the author but point at the new doc. # See https://docs.djangoproject.com/en/4.2/topics/db/queries/#copying-model-instances - author.pk = None - author.id = None - author._state.adding = True - author.document = doc - author.save() + documentauthor.pk = None + documentauthor.id = None + documentauthor._state.adding = True + documentauthor.document = doc + documentauthor.save() if draft: draft_events = [] @@ -629,43 +631,70 @@ def update_docs_from_rfc_index( ) rfc_published = True - def parse_relation_list(l): - res = [] - for x in l: - for a in Document.objects.filter(name=x.lower(), type_id="rfc"): - if a not in res: - res.append(a) - return res + def parse_relation_list(rel_list: list[str]) -> list[Document]: + return list( + Document.objects.filter( + name__in=[name.strip().lower() for name in rel_list], + type_id="rfc" + ) + ) - for x in parse_relation_list(obsoletes): - if not RelatedDocument.objects.filter( - source=doc, target=x, relationship=relationship_obsoletes + # Create missing obsoletes relations + docs_this_obsoletes = parse_relation_list(obsoletes) + for obs_doc in docs_this_obsoletes: + if not doc.relateddocument_set.filter( + target=obs_doc, relationship=relationship_obsoletes ): - r = RelatedDocument.objects.create( - source=doc, target=x, relationship=relationship_obsoletes + r = doc.relateddocument_set.create( + target=obs_doc, relationship=relationship_obsoletes ) rfc_changes.append( - "created {rel_name} relation between {src_name} and {tgt_name}".format( + "created {rel_name} relation between {src} and {tgt}".format( rel_name=r.relationship.name.lower(), - src_name=prettify_std_name(r.source.name), - tgt_name=prettify_std_name(r.target.name), + src=prettify_std_name(r.source.name), + tgt=prettify_std_name(r.target.name), ) ) + # Remove stale obsoletes relations + for r in doc.relateddocument_set.filter( + relationship=relationship_obsoletes + ).exclude(target_id__in=[d.pk for d in docs_this_obsoletes]): + r.delete() + rfc_changes.append( + "removed {rel_name} relation between {src} and {tgt}".format( + rel_name=r.relationship.name.lower(), + src=prettify_std_name(r.source.name), + tgt=prettify_std_name(r.target.name), + ) + ) - for x in parse_relation_list(updates): + docs_this_updates = parse_relation_list(updates) + for upd_doc in docs_this_updates: if not RelatedDocument.objects.filter( - source=doc, target=x, relationship=relationship_updates + source=doc, target=upd_doc, relationship=relationship_updates ): - r = RelatedDocument.objects.create( - source=doc, target=x, relationship=relationship_updates + r = doc.relateddocument_set.create( + target=upd_doc, relationship=relationship_updates ) rfc_changes.append( - "created {rel_name} relation between {src_name} and {tgt_name}".format( + "created {rel_name} relation between {src} and {tgt}".format( rel_name=r.relationship.name.lower(), - src_name=prettify_std_name(r.source.name), - tgt_name=prettify_std_name(r.target.name), + src=prettify_std_name(r.source.name), + tgt=prettify_std_name(r.target.name), ) ) + # Remove stale updates relations + for r in doc.relateddocument_set.filter( + relationship=relationship_updates + ).exclude(target_id__in=[d.pk for d in docs_this_updates]): + r.delete() + rfc_changes.append( + "removed {rel_name} relation between {src} and {tgt}".format( + rel_name=r.relationship.name.lower(), + src=prettify_std_name(r.source.name), + tgt=prettify_std_name(r.target.name), + ) + ) if also: # recondition also to have proper subseries document names: @@ -675,11 +704,11 @@ def parse_relation_list(l): subseries_slug = a[:3] if subseries_slug not in ["bcp", "std", "fyi"]: log(f"Unexpected 'also' relationship of {a} encountered for {doc}") - next + continue maybe_number = a[3:].strip() if not maybe_number.isdigit(): log(f"Unexpected 'also' subseries element identifier {a} encountered for {doc}") - next + continue else: subseries_number = int(maybe_number) conditioned_also.append(f"{subseries_slug}{subseries_number}") # Note the lack of leading zeros @@ -707,12 +736,27 @@ def parse_relation_list(l): subseries_doc.docevent_set.create(type="sync_from_rfc_editor", by=system, desc=f"Added {doc.name} to {subseries_doc.name}") rfc_events.append(doc.docevent_set.create(type="sync_from_rfc_editor", by=system, desc=f"Added {doc.name} to {subseries_doc.name}")) - for subdoc in doc.related_that("contains"): - if subdoc.name not in also: - assert(not first_sync_creating_subseries) - subseries_doc.relateddocument_set.filter(target=subdoc).delete() - rfc_events.append(doc.docevent_set.create(type="sync_from_rfc_editor", by=system, desc=f"Removed {doc.name} from {subseries_doc.name}")) - subseries_doc.docevent_set.create(type="sync_from_rfc_editor", by=system, desc=f"Removed {doc.name} from {subseries_doc.name}") + # Delete subseries relations that are no longer current. Use a transaction + # so we are sure we iterate over the same relations that we delete! + with transaction.atomic(): + stale_subseries_relations = doc.relations_that("contains").exclude( + source__name__in=also + ) + for stale_relation in stale_subseries_relations: + stale_subseries_doc = stale_relation.source + rfc_events.append( + doc.docevent_set.create( + type="sync_from_rfc_editor", + by=system, + desc=f"Removed {doc.name} from {stale_subseries_doc.name}", + ) + ) + stale_subseries_doc.docevent_set.create( + type="sync_from_rfc_editor", + by=system, + desc=f"Removed {doc.name} from {stale_subseries_doc.name}", + ) + stale_subseries_relations.delete() doc_errata = errata.get(f"RFC{rfc_number}", []) all_rejected = doc_errata and all( @@ -754,9 +798,9 @@ def parse_relation_list(l): ) doc.save_with_history(rfc_events) yield rfc_number, rfc_changes, doc, rfc_published # yield changes to the RFC - + if first_sync_creating_subseries: - # First - create the known subseries documents that have ghosted. + # First - create the known subseries documents that have ghosted. # The RFC editor (as of 31 Oct 2023) claims these subseries docs do not exist. # The datatracker, on the other hand, will say that the series doc currently contains no RFCs. for name in ["fyi17", "std1", "bcp12", "bcp113", "bcp66"]: @@ -769,7 +813,6 @@ def parse_relation_list(l): subseries_slug = name[:3] subseries_doc.docevent_set.create(type=f"{subseries_slug}_history_marker", by=system, desc=f"No history of this {subseries_slug.upper()} document is currently available in the datatracker before this point") - RelatedDocument.objects.filter( Q(originaltargetaliasname__startswith="bcp") | Q(originaltargetaliasname__startswith="std") | @@ -799,50 +842,4 @@ def parse_relation_list(l): ).update(document=F("subseries_target")) -def post_approved_draft(url, name): - """Post an approved draft to the RFC Editor so they can retrieve - the data from the Datatracker and start processing it. Returns - response and error (empty string if no error).""" - - if settings.SERVER_MODE != "production": - log(f"In production, would have posted RFC-Editor notification of approved I-D '{name}' to '{url}'") - return "", "" - - # HTTP basic auth - username = "dtracksync" - password = settings.RFC_EDITOR_SYNC_PASSWORD - headers = { - "Content-type": "application/x-www-form-urlencoded", - "Accept": "text/plain", - "Authorization": "Basic %s" % force_str(base64.encodebytes(smart_bytes("%s:%s" % (username, password)))).replace("\n", ""), - } - - log("Posting RFC-Editor notification of approved Internet-Draft '%s' to '%s'" % (name, url)) - text = error = "" - - try: - r = requests.post( - url, - headers=headers, - data=smart_bytes(urlencode({ 'draft': name })), - timeout=settings.DEFAULT_REQUESTS_TIMEOUT, - ) - - log("RFC-Editor notification result for Internet-Draft '%s': %s:'%s'" % (name, r.status_code, r.text)) - - if r.status_code != 200: - raise RuntimeError("Status code is not 200 OK (it's %s)." % r.status_code) - - if force_str(r.text) != "OK": - raise RuntimeError('Response is not "OK" (it\'s "%s").' % r.text) - - except Exception as e: - # catch everything so we don't leak exceptions, convert them - # into string instead - msg = "Exception on RFC-Editor notification for Internet-Draft '%s': %s: %s" % (name, type(e), str(e)) - log(msg) - if settings.SERVER_MODE == 'test': - debug.say(msg) - error = str(e) - return text, error diff --git a/ietf/sync/rfcindex.py b/ietf/sync/rfcindex.py new file mode 100644 index 0000000000..be55a6866e --- /dev/null +++ b/ietf/sync/rfcindex.py @@ -0,0 +1,821 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +import datetime +import json +import shutil +from collections import defaultdict +from collections.abc import Container, Iterable +from dataclasses import dataclass +from itertools import chain +from operator import attrgetter, itemgetter +from pathlib import Path +from textwrap import fill +from urllib.parse import urljoin + +from django.conf import settings +from django.core.files.base import ContentFile +from django.db.models import Q +from lxml import etree + +from django.core.files.storage import storages +from django.db import models +from django.db.models.functions import Substr, Cast +from django.template.loader import render_to_string +from django.utils import timezone + +from ietf.doc.models import Document +from ietf.name.models import StdLevelName +from ietf.utils.log import log +from ietf.utils.models import DirtyBits + +FORMATS_FOR_INDEX = ["txt", "html", "pdf", "xml", "ps"] +SS_TXT_MARGIN = 3 +SS_TXT_CUE_COL_WIDTH = 14 + + +def format_rfc_number(n): + """Format an RFC number (or subseries doc number) + + Set settings.RFCINDEX_MATCH_LEGACY_XML=True for the legacy (leading-zero) format. + That is for debugging only - tests will fail. + """ + if getattr(settings, "RFCINDEX_MATCH_LEGACY_XML", False): + return format(n, "04") + else: + return format(n) + + +def errata_url(rfc: Document): + return urljoin(settings.RFC_EDITOR_ERRATA_BASE_URL + "/", f"rfc{rfc.rfc_number}") + + +def save_to_red_bucket(filename: str, content: str | bytes): + red_bucket = storages["red_bucket"] + bucket_path = str(Path(getattr(settings, "RFCINDEX_OUTPUT_PATH", "")) / filename) + if getattr(settings, "RFCINDEX_DELETE_THEN_WRITE", True): + # Django 4.2's FileSystemStorage does not support allow_overwrite. + red_bucket.delete(bucket_path) + red_bucket.save( + bucket_path, + ContentFile(content if isinstance(content, bytes) else content.encode("utf-8")), + ) + log(f"Saved {bucket_path} in red_bucket storage") + + +def save_to_filesystem( + filename: str, content: str | bytes, subdirs: Iterable[str] = () +): + """Save contents to the RFC_PATH in the filesystem + + Always saves directly to settings.RFC_PATH/filename. Additionally saves a copy + to settings.RFC_PATH/subdir/filename for each entry in subdirs. Uses shutil.copy2 + to create the copies, which will preserve mtime and other metadata between copies. + """ + rfc_path = Path(settings.RFC_PATH) + dest_path = rfc_path / filename + dest_path.write_bytes( + content if isinstance(content, bytes) else content.encode("utf-8") + ) + for subdir in subdirs: + (rfc_path / subdir).mkdir(parents=False, exist_ok=True) + shutil.copy2(dest_path, rfc_path / subdir / filename) + + +@dataclass +class UnusableRfcNumber: + rfc_number: int + comment: str + + +def get_unusable_rfc_numbers() -> list[UnusableRfcNumber]: + FILENAME = "unusable-rfc-numbers.json" + bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME) + try: + with storages["red_bucket"].open(bucket_path) as urn_file: + records = json.load(urn_file) + except FileNotFoundError: + if settings.SERVER_MODE == "development": + log( + f"Unable to open {bucket_path} in red_bucket storage. This is okay in dev " + "but generated rfc-index will not agree with RFC Editor values." + ) # pragma: no cover + return [] # pragma: no cover + log(f"Error: unable to open {bucket_path} in red_bucket storage") + raise + except json.JSONDecodeError: + log(f"Error: unable to parse {bucket_path} in red_bucket storage") + if settings.SERVER_MODE == "development": + return [] # pragma: no cover + raise + assert all(isinstance(record["number"], int) for record in records) + assert all(isinstance(record["comment"], str) for record in records) + return [ + UnusableRfcNumber(rfc_number=record["number"], comment=record["comment"]) + for record in sorted(records, key=itemgetter("number")) + ] + + +def get_april1_rfc_numbers() -> Container[int]: + FILENAME = "april-first-rfc-numbers.json" + bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME) + try: + with storages["red_bucket"].open(bucket_path) as urn_file: + records = json.load(urn_file) + except FileNotFoundError: + if settings.SERVER_MODE == "development": + log( + f"Unable to open {bucket_path} in red_bucket storage. This is okay in dev " + "but generated rfc-index will not agree with RFC Editor values." + ) # pragma: no cover + return [] # pragma: no cover + log(f"Error: unable to open {bucket_path} in red_bucket storage") + raise + except json.JSONDecodeError: + log(f"Error: unable to parse {bucket_path} in red_bucket storage") + if settings.SERVER_MODE == "development": + return [] # pragma: no cover + raise + assert all(isinstance(record, int) for record in records) + return records + + +def get_publication_std_levels() -> dict[int, StdLevelName]: + FILENAME = "publication-std-levels.json" + bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME) + values: dict[int, StdLevelName] = {} + try: + with storages["red_bucket"].open(bucket_path) as urn_file: + records = json.load(urn_file) + except FileNotFoundError: + if settings.SERVER_MODE == "development": + log( + f"Unable to open {bucket_path} in red_bucket storage. This is okay in dev " + "but generated rfc-index will not agree with RFC Editor values." + ) # pragma: no cover + # intentionally fall through instead of return here + else: + log(f"Error: unable to open {bucket_path} in red_bucket storage") + raise + except json.JSONDecodeError: + log(f"Error: unable to parse {bucket_path} in red_bucket storage") + if settings.SERVER_MODE != "development": + raise + else: + assert all(isinstance(record["number"], int) for record in records) + values = { + record["number"]: StdLevelName.objects.get( + slug=record["publication_std_level"] + ) + for record in records + } + # defaultdict to return "unknown" for any missing values + unknown_std_level = StdLevelName.objects.get(slug="unkn") + return defaultdict(lambda: unknown_std_level, values) + + +def format_ordering(rfc_number): + if rfc_number < settings.FIRST_V3_RFC: + ordering = ["txt", "ps", "pdf", "html", "xml"] + else: + ordering = ["html", "txt", "ps", "pdf", "xml"] + return ordering.index # return the method + + +def get_rfc_text_index_entries(): + """Returns RFC entries for rfc-index.txt""" + entries = [] + april1_rfc_numbers = get_april1_rfc_numbers() + published_rfcs = Document.objects.filter(type_id="rfc").order_by("rfc_number") + rfcs = sorted( + chain(published_rfcs, get_unusable_rfc_numbers()), key=attrgetter("rfc_number") + ) + for rfc in rfcs: + if isinstance(rfc, UnusableRfcNumber): + entries.append(f"{format_rfc_number(rfc.rfc_number)} Not Issued.") + else: + assert isinstance(rfc, Document) + authors = ", ".join( + author.format_for_titlepage() for author in rfc.rfcauthor_set.all() + ) + published_at = rfc.pub_date() + date = ( + published_at.strftime("1 %B %Y") + if rfc.rfc_number in april1_rfc_numbers + else published_at.strftime("%B %Y") + ) + + # formats + formats = ", ".join( + sorted( + [ + format["fmt"] + for format in rfc.formats() + if format["fmt"] in FORMATS_FOR_INDEX + ], + key=format_ordering(rfc.rfc_number), + ) + ).upper() + + # obsoletes + obsoletes = "" + obsoletes_documents = sorted( + rfc.related_that_doc("obs"), + key=attrgetter("rfc_number"), + ) + if len(obsoletes_documents) > 0: + obsoletes_names = ", ".join( + f"RFC{format_rfc_number(doc.rfc_number)}" + for doc in obsoletes_documents + ) + obsoletes = f" (Obsoletes {obsoletes_names})" + + # obsoleted by + obsoleted_by = "" + obsoleted_by_documents = sorted( + rfc.related_that("obs"), + key=attrgetter("rfc_number"), + ) + if len(obsoleted_by_documents) > 0: + obsoleted_by_names = ", ".join( + f"RFC{format_rfc_number(doc.rfc_number)}" + for doc in obsoleted_by_documents + ) + obsoleted_by = f" (Obsoleted by {obsoleted_by_names})" + + # updates + updates = "" + updates_documents = sorted( + rfc.related_that_doc("updates"), + key=attrgetter("rfc_number"), + ) + if len(updates_documents) > 0: + updates_names = ", ".join( + f"RFC{format_rfc_number(doc.rfc_number)}" + for doc in updates_documents + ) + updates = f" (Updates {updates_names})" + + # updated by + updated_by = "" + updated_by_documents = sorted( + rfc.related_that("updates"), + key=attrgetter("rfc_number"), + ) + if len(updated_by_documents) > 0: + updated_by_names = ", ".join( + f"RFC{format_rfc_number(doc.rfc_number)}" + for doc in updated_by_documents + ) + updated_by = f" (Updated by {updated_by_names})" + + doc_relations = f"{obsoletes}{obsoleted_by}{updates}{updated_by} " + + # subseries + subseries = ",".join( + f"{container.type.slug}{format_rfc_number(int(container.name[3:]))}" + for container in rfc.part_of() + ).upper() + if subseries: + subseries = f"(Also {subseries}) " + + entry = fill( + ( + f"{format_rfc_number(rfc.rfc_number)} {rfc.title}. {authors}. {date}. " + f"(Format: {formats}){doc_relations}{subseries}" + f"(Status: {str(rfc.std_level).upper()}) " + f"(DOI: {rfc.doi})" + ), + width=73, + subsequent_indent=" " * 5, + ) + entries.append(entry) + + return entries + + +def subseries_text_line(line, first=False): + """Return subseries text entry line""" + indent = " " * SS_TXT_CUE_COL_WIDTH + if first: + initial_indent = " " * SS_TXT_MARGIN + else: + initial_indent = indent + return fill( + line, + initial_indent=initial_indent, + subsequent_indent=indent, + width=80, + break_on_hyphens=False, + ) + + +def get_bcp_text_index_entries(): + """Returns BCP entries for bcp-index.txt""" + entries = [] + + highest_bcp_number = ( + Document.objects.filter(type_id="bcp") + .annotate( + number=Cast( + Substr("name", 4, None), + output_field=models.IntegerField(), + ) + ) + .order_by("-number") + .first() + .number + ) + + for bcp_number in range(1, highest_bcp_number + 1): + bcp_name = f"BCP{bcp_number}" + bcp = Document.objects.filter(type_id="bcp", name=f"{bcp_name.lower()}").first() + + if bcp: + entry = subseries_text_line( + ( + f"[{bcp_name}]" + f"{' ' * (SS_TXT_CUE_COL_WIDTH - len(bcp_name) - 2 - SS_TXT_MARGIN)}" + f"Best Current Practice {bcp_number}," + ), + first=True, + ) + entry += "\n" + entry += subseries_text_line( + f"<{settings.RFC_EDITOR_INFO_BASE_URL}{bcp_name.lower()}>." + ) + entry += "\n" + entry += subseries_text_line( + "At the time of writing, this BCP comprises the following:" + ) + entry += "\n\n" + rfcs = sorted(bcp.contains(), key=lambda x: x.rfc_number) + for rfc in rfcs: + authors = ", ".join( + author.format_for_titlepage() for author in rfc.rfcauthor_set.all() + ) + entry += subseries_text_line( + ( + f'{authors}, "{rfc.title}", BCP¶{bcp_number}, RFC¶{rfc.rfc_number}, ' + f"DOI¶{rfc.doi}, {rfc.pub_date().strftime('%B %Y')}, " + f"<{settings.RFC_EDITOR_INFO_BASE_URL}rfc{rfc.rfc_number}>." + ) + ).replace("¶", " ") + entry += "\n\n" + else: + entry = subseries_text_line( + ( + f"[{bcp_name}]" + f"{' ' * (SS_TXT_CUE_COL_WIDTH - len(bcp_name) - 2 - SS_TXT_MARGIN)}" + f"Best Current Practice {bcp_number} currently contains no RFCs" + ), + first=True, + ) + entries.append(entry) + return entries + + +def get_std_text_index_entries(): + """Returns STD entries for std-index.txt""" + entries = [] + + highest_std_number = ( + Document.objects.filter(type_id="std") + .annotate( + number=Cast( + Substr("name", 4, None), + output_field=models.IntegerField(), + ) + ) + .order_by("-number") + .first() + .number + ) + + for std_number in range(1, highest_std_number + 1): + std_name = f"STD{std_number}" + std = Document.objects.filter(type_id="std", name=f"{std_name.lower()}").first() + + if std and std.contains(): + entry = subseries_text_line( + ( + f"[{std_name}]" + f"{' ' * (SS_TXT_CUE_COL_WIDTH - len(std_name) - 2 - SS_TXT_MARGIN)}" + f"Internet Standard {std_number}," + ), + first=True, + ) + entry += "\n" + entry += subseries_text_line( + f"<{settings.RFC_EDITOR_INFO_BASE_URL}{std_name.lower()}>." + ) + entry += "\n" + entry += subseries_text_line( + "At the time of writing, this STD comprises the following:" + ) + entry += "\n\n" + rfcs = sorted(std.contains(), key=lambda x: x.rfc_number) + for rfc in rfcs: + authors = ", ".join( + author.format_for_titlepage() for author in rfc.rfcauthor_set.all() + ) + entry += subseries_text_line( + ( + f'{authors}, "{rfc.title}", STD¶{std_number}, RFC¶{rfc.rfc_number}, ' + f"DOI¶{rfc.doi}, {rfc.pub_date().strftime('%B %Y')}, " + f"<{settings.RFC_EDITOR_INFO_BASE_URL}rfc{rfc.rfc_number}>." + ) + ).replace("¶", " ") + entry += "\n\n" + else: + entry = subseries_text_line( + ( + f"[{std_name}]" + f"{' ' * (SS_TXT_CUE_COL_WIDTH - len(std_name) - 2 - SS_TXT_MARGIN)}" + f"Internet Standard {std_number} currently contains no RFCs" + ), + first=True, + ) + entries.append(entry) + return entries + + +def get_fyi_text_index_entries(): + """Returns FYI entries for fyi-index.txt""" + entries = [] + + highest_fyi_number = ( + Document.objects.filter(type_id="fyi") + .annotate( + number=Cast( + Substr("name", 4, None), + output_field=models.IntegerField(), + ) + ) + .order_by("-number") + .first() + .number + ) + + for fyi_number in range(1, highest_fyi_number + 1): + fyi_name = f"FYI{fyi_number}" + fyi = Document.objects.filter(type_id="fyi", name=f"{fyi_name.lower()}").first() + + if fyi and fyi.contains(): + entry = subseries_text_line( + ( + f"[{fyi_name}]" + f"{' ' * (SS_TXT_CUE_COL_WIDTH - len(fyi_name) - 2 - SS_TXT_MARGIN)}" + f"For Your Information {fyi_number}," + ), + first=True, + ) + entry += "\n" + entry += subseries_text_line( + f"<{settings.RFC_EDITOR_INFO_BASE_URL}{fyi_name.lower()}>." + ) + entry += "\n" + entry += subseries_text_line( + "At the time of writing, this FYI comprises the following:" + ) + entry += "\n\n" + rfcs = sorted(fyi.contains(), key=lambda x: x.rfc_number) + for rfc in rfcs: + authors = ", ".join( + author.format_for_titlepage() for author in rfc.rfcauthor_set.all() + ) + entry += subseries_text_line( + ( + f'{authors}, "{rfc.title}", FYI¶{fyi_number}, RFC¶{rfc.rfc_number}, ' + f"DOI¶{rfc.doi}, {rfc.pub_date().strftime('%B %Y')}, " + f"<{settings.RFC_EDITOR_INFO_BASE_URL}rfc{rfc.rfc_number}>." + ) + ).replace("¶", " ") + entry += "\n\n" + else: + entry = subseries_text_line( + ( + f"[{fyi_name}]" + f"{' ' * (SS_TXT_CUE_COL_WIDTH - len(fyi_name) - 2 - SS_TXT_MARGIN)}" + f"For Your Information {fyi_number} currently contains no RFCs" + ), + first=True, + ) + entries.append(entry) + return entries + + +def add_subseries_xml_index_entries(rfc_index, ss_type, include_all=False): + """Add subseries entries for rfc-index.xml""" + # subseries docs annotated with numeric number + ss_docs = list( + Document.objects.filter(type_id=ss_type) + .annotate( + number=Cast( + Substr("name", 4, None), + output_field=models.IntegerField(), + ) + ) + .order_by("-number") + ) + if len(ss_docs) == 0: + return # very much not expected + highest_number = ss_docs[0].number + for ss_number in range(1, highest_number + 1): + if ss_docs[-1].number == ss_number: + this_ss_doc = ss_docs.pop() + contained_rfcs = this_ss_doc.contains() + else: + contained_rfcs = [] + if len(contained_rfcs) == 0 and not include_all: + continue + entry = etree.SubElement(rfc_index, f"{ss_type}-entry") + etree.SubElement( + entry, "doc-id" + ).text = f"{ss_type.upper()}{format_rfc_number(ss_number)}" + if len(contained_rfcs) > 0: + is_also = etree.SubElement(entry, "is-also") + for rfc in sorted(contained_rfcs, key=attrgetter("rfc_number")): + etree.SubElement( + is_also, "doc-id" + ).text = f"RFC{format_rfc_number(rfc.rfc_number)}" + + +def add_related_xml_index_entries(root: etree.Element, rfc: Document, tag: str): + relation_getter = { + "obsoletes": lambda doc: doc.related_that_doc("obs"), + "obsoleted-by": lambda doc: doc.related_that("obs"), + "updates": lambda doc: doc.related_that_doc("updates"), + "updated-by": lambda doc: doc.related_that("updates"), + } + related_docs = sorted( + relation_getter[tag](rfc), + key=attrgetter("rfc_number"), + ) + if len(related_docs) > 0: + element = etree.SubElement(root, tag) + for doc in related_docs: + etree.SubElement( + element, "doc-id" + ).text = f"RFC{format_rfc_number(doc.rfc_number)}" + + +def add_rfc_xml_index_entries(rfc_index): + """Add RFC entries for rfc-index.xml""" + entries = [] + april1_rfc_numbers = get_april1_rfc_numbers() + publication_statuses = get_publication_std_levels() + + published_rfcs = Document.objects.filter(type_id="rfc").order_by("rfc_number") + + # Iterators for unpublished and published, both sorted by number + unpublished_iter = iter(get_unusable_rfc_numbers()) + published_iter = iter(published_rfcs) + + # Prime the next_* values + next_unpublished = next(unpublished_iter, None) + next_published = next(published_iter, None) + + while next_published is not None or next_unpublished is not None: + if next_unpublished is not None and ( + next_published is None + or next_unpublished.rfc_number < next_published.rfc_number + ): + entry = etree.SubElement(rfc_index, "rfc-not-issued-entry") + etree.SubElement( + entry, "doc-id" + ).text = f"RFC{format_rfc_number(next_unpublished.rfc_number)}" + entries.append(entry) + next_unpublished = next(unpublished_iter, None) + continue + + rfc = next_published # hang on to this + next_published = next(published_iter, None) # prep for next iteration + entry = etree.SubElement(rfc_index, "rfc-entry") + + etree.SubElement( + entry, "doc-id" + ).text = f"RFC{format_rfc_number(rfc.rfc_number)}" + etree.SubElement(entry, "title").text = rfc.title + + for author in rfc.rfcauthor_set.all(): + author_element = etree.SubElement(entry, "author") + etree.SubElement(author_element, "name").text = author.titlepage_name + if author.is_editor: + etree.SubElement(author_element, "title").text = "Editor" + + date = etree.SubElement(entry, "date") + published_at = rfc.pub_date() + etree.SubElement(date, "month").text = published_at.strftime("%B") + if rfc.rfc_number in april1_rfc_numbers: + etree.SubElement(date, "day").text = str(published_at.day) + etree.SubElement(date, "year").text = str(published_at.year) + + format_ = etree.SubElement(entry, "format") + fmts = [ff["fmt"] for ff in rfc.formats() if ff["fmt"] in FORMATS_FOR_INDEX] + for fmt in sorted(fmts, key=format_ordering(rfc.rfc_number)): + match_legacy = getattr(settings, "RFCINDEX_MATCH_LEGACY_XML", False) + etree.SubElement(format_, "file-format").text = ( + "ASCII" if match_legacy and fmt == "txt" else fmt.upper() + ) + + etree.SubElement(entry, "page-count").text = str(rfc.pages) + + if len(rfc.keywords) > 0: + keywords = etree.SubElement(entry, "keywords") + for keyword in rfc.keywords: + etree.SubElement(keywords, "kw").text = keyword.strip() + + if rfc.abstract: + abstract = etree.SubElement(entry, "abstract") + for paragraph in rfc.abstract.split("\n\n"): + etree.SubElement(abstract, "p").text = paragraph.strip() + + draft = rfc.came_from_draft() + if draft is not None: + etree.SubElement(entry, "draft").text = f"{draft.name}-{draft.rev}" + + part_of_documents = rfc.part_of() + if len(part_of_documents) > 0: + is_also = etree.SubElement(entry, "is-also") + for doc in part_of_documents: + etree.SubElement(is_also, "doc-id").text = doc.name.upper() + + add_related_xml_index_entries(entry, rfc, "obsoletes") + add_related_xml_index_entries(entry, rfc, "obsoleted-by") + add_related_xml_index_entries(entry, rfc, "updates") + add_related_xml_index_entries(entry, rfc, "updated-by") + + etree.SubElement(entry, "current-status").text = rfc.std_level.name.upper() + etree.SubElement(entry, "publication-status").text = publication_statuses[ + rfc.rfc_number + ].name.upper() + etree.SubElement(entry, "stream").text = ( + "INDEPENDENT" if rfc.stream_id == "ise" else rfc.stream.name + ) + + # Add area / wg_acronym + if rfc.stream_id == "ietf": + if rfc.group.type_id in ["individ", "area"]: + etree.SubElement(entry, "wg_acronym").text = "NON WORKING GROUP" + else: + if rfc.area is not None: + etree.SubElement(entry, "area").text = rfc.area.acronym + if rfc.group: + etree.SubElement(entry, "wg_acronym").text = rfc.group.acronym + + if rfc.tags.filter(slug="errata").exists(): + etree.SubElement(entry, "errata-url").text = errata_url(rfc) + etree.SubElement(entry, "doi").text = rfc.doi + entries.append(entry) + + +def create_rfc_txt_index(): + """Create text index of published documents""" + DATE_FMT = "%m/%d/%Y" + created_on = timezone.now().strftime(DATE_FMT) + log("Creating rfc-index.txt") + index = render_to_string( + "sync/rfc-index.txt", + { + "created_on": created_on, + "rfcs": get_rfc_text_index_entries(), + }, + ) + filename = "rfc-index.txt" + save_to_red_bucket(filename, index) + save_to_filesystem(filename, index) + + +def create_rfc_xml_index(): + """Create XML index of published documents""" + XSI_NAMESPACE = "http://www.w3.org/2001/XMLSchema-instance" + XSI = "{" + XSI_NAMESPACE + "}" + + log("Creating rfc-index.xml") + rfc_index = etree.Element( + "rfc-index", + nsmap={ + None: "https://www.rfc-editor.org/rfc-index", + "xsi": XSI_NAMESPACE, + }, + attrib={ + XSI + "schemaLocation": ( + "https://www.rfc-editor.org/rfc-index " + "https://www.rfc-editor.org/rfc-index.xsd" + ), + }, + ) + + # add data + add_subseries_xml_index_entries(rfc_index, "bcp", include_all=True) + add_subseries_xml_index_entries(rfc_index, "fyi") + add_rfc_xml_index_entries(rfc_index) + add_subseries_xml_index_entries(rfc_index, "std") + + # make it pretty + pretty_index = etree.tostring( + rfc_index, + encoding="utf-8", + xml_declaration=True, + pretty_print=4, + ) + filename = "rfc-index.xml" + save_to_red_bucket(filename, pretty_index) + save_to_filesystem(filename, pretty_index) + + +def create_bcp_txt_index(): + """Create text index of BCPs""" + DATE_FMT = "%m/%d/%Y" + created_on = timezone.now().strftime(DATE_FMT) + log("Creating bcp-index.txt") + index = render_to_string( + "sync/bcp-index.txt", + { + "created_on": created_on, + "bcps": get_bcp_text_index_entries(), + }, + ) + filename = "bcp-index.txt" + save_to_red_bucket(filename, index) + save_to_filesystem(filename, index, ["bcp"]) + + +def create_std_txt_index(): + """Create text index of STDs""" + DATE_FMT = "%m/%d/%Y" + created_on = timezone.now().strftime(DATE_FMT) + log("Creating std-index.txt") + index = render_to_string( + "sync/std-index.txt", + { + "created_on": created_on, + "stds": get_std_text_index_entries(), + }, + ) + filename = "std-index.txt" + save_to_red_bucket(filename, index) + save_to_filesystem(filename, index, ["std"]) + + +def create_fyi_txt_index(): + """Create text index of FYIs""" + DATE_FMT = "%m/%d/%Y" + created_on = timezone.now().strftime(DATE_FMT) + log("Creating fyi-index.txt") + index = render_to_string( + "sync/fyi-index.txt", + { + "created_on": created_on, + "fyis": get_fyi_text_index_entries(), + }, + ) + filename = "fyi-index.txt" + save_to_red_bucket(filename, index) + save_to_filesystem(filename, index, ["fyi"]) + + +## DirtyBits management for the RFC index + +RFCINDEX_SLUG = DirtyBits.Slugs.RFCINDEX + + +def mark_rfcindex_as_dirty(): + _, created = DirtyBits.objects.update_or_create( + slug=RFCINDEX_SLUG, defaults={"dirty_time": timezone.now()} + ) + if created: + log(f"Created DirtyBits(slug='{RFCINDEX_SLUG}')") + + +def mark_rfcindex_as_processed(when: datetime.datetime): + n_updated = DirtyBits.objects.filter( + Q(processed_time__isnull=True) | Q(processed_time__lt=when), + slug=RFCINDEX_SLUG, + ).update(processed_time=when) + if n_updated > 0: + log(f"processed_time is now {when.isoformat()}") + else: + log("processed_time not updated, no matching record found") + + +def rfcindex_is_dirty(): + """Does the rfc index need to be updated?""" + dirty_work, created = DirtyBits.objects.get_or_create( + slug=RFCINDEX_SLUG, defaults={"dirty_time": timezone.now()} + ) + if created: + log(f"Created DirtyBits(slug='{RFCINDEX_SLUG}')") + display_processed_time = ( + dirty_work.processed_time.isoformat() + if dirty_work.processed_time is not None + else "never" + ) + log( + f"DirtyBits(slug='{RFCINDEX_SLUG}'): " + f"dirty_time={dirty_work.dirty_time.isoformat()} " + f"processed_time={display_processed_time}" + ) + return ( + dirty_work.processed_time is None + or dirty_work.dirty_time >= dirty_work.processed_time + ) diff --git a/ietf/sync/tasks.py b/ietf/sync/tasks.py index 53e23d7913..3808fb1db5 100644 --- a/ietf/sync/tasks.py +++ b/ietf/sync/tasks.py @@ -1,9 +1,11 @@ -# Copyright The IETF Trust 2024, All Rights Reserved +# Copyright The IETF Trust 2024-2026, All Rights Reserved # # Celery task definitions # import datetime import io +from pathlib import Path +from tempfile import NamedTemporaryFile import requests from celery import shared_task @@ -11,9 +13,28 @@ from django.conf import settings from django.utils import timezone +from ietf.doc.models import DocEvent, DocTagName, Document, RelatedDocument, RpcAssignmentDocEvent, State +from ietf.doc.tasks import rebuild_reference_relations_task +from ietf.doc.utils import add_state_change_event, new_state_change_event, update_action_holders +from ietf.person.models import Person +from ietf.utils.mail import send_mail_text from ietf.sync import iana from ietf.sync import rfceditor +from ietf.sync.errata import ( + errata_are_dirty, + mark_errata_as_processed, + update_errata_from_rfceditor, +) from ietf.sync.rfceditor import MIN_QUEUE_RESULTS, parse_queue, update_drafts_from_queue +from ietf.sync.rfcindex import ( + create_bcp_txt_index, + create_fyi_txt_index, + create_rfc_txt_index, + create_rfc_xml_index, + create_std_txt_index, + rfcindex_is_dirty, mark_rfcindex_as_processed, mark_rfcindex_as_dirty, +) +from ietf.sync.utils import build_from_file_content, load_rfcs_into_blobdb, rsync_helper from ietf.utils import log from ietf.utils.timezone import date_today @@ -21,13 +42,13 @@ @shared_task def rfc_editor_index_update_task(full_index=False): """Update metadata from the RFC index - + Default is to examine only changes in the past 365 days. Call with full_index=True to update the full RFC index. - + According to comments on the original script, a year's worth took about 20s on production as of August 2022 - + The original rfc-editor-index-update script had a long-disabled provision for running the rebuild_reference_relations scripts after the update. That has not been brought over at all because it should be implemented as its own task if it is needed. @@ -45,7 +66,7 @@ def rfc_editor_index_update_task(full_index=False): timeout=30, # seconds ) except requests.Timeout as exc: - log.log(f'GET request timed out retrieving RFC editor index: {exc}') + log.log(f"GET request timed out retrieving RFC editor index: {exc}") return # failed rfc_index_xml = response.text index_data = rfceditor.parse_index(io.StringIO(rfc_index_xml)) @@ -55,20 +76,25 @@ def rfc_editor_index_update_task(full_index=False): timeout=30, # seconds ) except requests.Timeout as exc: - log.log(f'GET request timed out retrieving RFC editor errata: {exc}') + log.log(f"GET request timed out retrieving RFC editor errata: {exc}") return # failed - errata_data = response.json() + errata_data = response.json() if len(index_data) < rfceditor.MIN_INDEX_RESULTS: log.log("Not enough index entries, only %s" % len(index_data)) return # failed if len(errata_data) < rfceditor.MIN_ERRATA_RESULTS: log.log("Not enough errata entries, only %s" % len(errata_data)) return # failed + newly_published = set() for rfc_number, changes, doc, rfc_published in rfceditor.update_docs_from_rfc_index( index_data, errata_data, skip_older_than_date=skip_date ): for c in changes: log.log("RFC%s, %s: %s" % (rfc_number, doc.name, c)) + if rfc_published: + newly_published.add(rfc_number) + if len(newly_published) > 0: + rsync_rfcs_from_rfceditor_task.delay(list(newly_published)) @shared_task @@ -85,15 +111,15 @@ def rfc_editor_queue_updates_task(): drafts, warnings = parse_queue(io.StringIO(response.text)) for w in warnings: log.log(f"Warning: {w}") - + if len(drafts) < MIN_QUEUE_RESULTS: log.log("Not enough results, only %s" % len(drafts)) return # failed - + changed, warnings = update_drafts_from_queue(drafts) for w in warnings: log.log(f"Warning: {w}") - + for c in changed: log.log(f"Updated {c}") @@ -109,9 +135,11 @@ def iana_changes_update_task(): MAX_INTERVAL_ACCEPTED_BY_IANA = datetime.timedelta(hours=23) start = ( - timezone.now() - - datetime.timedelta(hours=23) - + datetime.timedelta(seconds=CLOCK_SKEW_COMPENSATION,) + timezone.now() + - datetime.timedelta(hours=23) + + datetime.timedelta( + seconds=CLOCK_SKEW_COMPENSATION, + ) ) end = start + datetime.timedelta(hours=23) @@ -122,7 +150,9 @@ def iana_changes_update_task(): # requests if necessary text = iana.fetch_changes_json( - settings.IANA_SYNC_CHANGES_URL, t, min(end, t + MAX_INTERVAL_ACCEPTED_BY_IANA) + settings.IANA_SYNC_CHANGES_URL, + t, + min(end, t + MAX_INTERVAL_ACCEPTED_BY_IANA), ) log.log(f"Retrieved the JSON: {text}") @@ -148,10 +178,10 @@ def iana_protocols_update_task(): # "this needs to be the date where this tool is first deployed" in the original # iana-protocols-updates script)" rfc_must_published_later_than = datetime.datetime( - 2012, - 11, - 26, - tzinfo=datetime.timezone.utc, + 2012, + 11, + 26, + tzinfo=datetime.UTC, ) try: @@ -160,17 +190,17 @@ def iana_protocols_update_task(): timeout=30, ) except requests.Timeout as exc: - log.log(f'GET request timed out retrieving IANA protocols page: {exc}') + log.log(f"GET request timed out retrieving IANA protocols page: {exc}") return rfc_numbers = iana.parse_protocol_page(response.text) def batched(l, n): """Split list l up in batches of max size n. - + For Python 3.12 or later, replace this with itertools.batched() """ - return (l[i:i + n] for i in range(0, len(l), n)) + return (l[i : i + n] for i in range(0, len(l), n)) for batch in batched(rfc_numbers, 100): updated = iana.update_rfc_log_from_protocol_page( @@ -180,3 +210,262 @@ def batched(l, n): for d in updated: log.log("Added history entry for %s" % d.display_name()) + + +@shared_task +def fix_subseries_docevents_task(): + """Repairs DocEvents related to bugs around removing docs from subseries + + Removes bogus and repairs the date of non-bogus DocEvents + about removing RFCs from subseries + + This is designed to be a one-shot task that should be removed + after running it. It is intended to be safe if it runs more than once. + """ + log.log("Repairing DocEvents related to bugs around removing docs from subseries") + bogus_event_descs = [ + "Removed rfc8499 from bcp218", + "Removed rfc7042 from bcp184", + "Removed rfc9499 from bcp238", + "Removed rfc5033 from std74", + "Removed rfc3228 from bcp55", + "Removed rfc8109 from std85", + ] + DocEvent.objects.filter( + type="sync_from_rfc_editor", desc__in=bogus_event_descs + ).delete() + needs_moment_fix = [ + "Removed rfc8499 from bcp219", + "Removed rfc7042 from bcp141", + "Removed rfc5033 from bcp133", + "Removed rfc3228 from bcp57", + ] + # Assumptions (which have been manually verified): + # 1) each of the above RFCs is obsoleted by exactly one other RFC + # 2) each of the obsoleting RFCs has exactly one published_rfc docevent + for desc in needs_moment_fix: + obsoleted_rfc_name = desc.split(" ")[1] + obsoleting_rfc = RelatedDocument.objects.get( + relationship_id="obs", target__name=obsoleted_rfc_name + ).source + obsoleting_time = obsoleting_rfc.docevent_set.get(type="published_rfc").time + DocEvent.objects.filter(type="sync_from_rfc_editor", desc=desc).update( + time=obsoleting_time + ) + + +@shared_task +def rsync_rfcs_from_rfceditor_task(rfc_numbers: list[int]): + log.log(f"Rsyncing rfcs from rfc-editor: {rfc_numbers}") + from_file = None + with NamedTemporaryFile(mode="w", delete_on_close=False) as fp: + fp.write(build_from_file_content(rfc_numbers)) + fp.close() + from_file = Path(fp.name) + rsync_helper( + [ + "-a", + "--ignore-existing", + f"--include-from={from_file}", + "--exclude=*", + "rsync.rfc-editor.org::rfcs/", + f"{settings.RFC_PATH}", + ] + ) + load_rfcs_into_blobdb(rfc_numbers) + + rebuild_reference_relations_task.delay([f"rfc{num}" for num in rfc_numbers]) + + +@shared_task +def load_rfcs_into_blobdb_task(start: int, end: int): + """Move file content for rfcs from rfc{start} to rfc{end} inclusive + + As this is expected to be removed once the blobdb is populated, it + will truncate its work to a coded max end. + This will not overwrite any existing blob content, and will only + log a small complaint if asked to load a non-exsiting RFC. + """ + # Protect us from ourselves + if end < start: + return + if start < 1: + start = 1 + if end > 11000: # Arbitrarily chosen + end = 11000 + load_rfcs_into_blobdb(list(range(start, end + 1))) + + +@shared_task +def update_errata_from_rfceditor_task(): + if errata_are_dirty(): + # new_processed_time is the *start* of processing so that any changes after + # this point will trigger another refresh + new_processed_time = timezone.now() + update_errata_from_rfceditor() + mark_errata_as_processed(new_processed_time) + mark_rfcindex_as_dirty() # ensure any changes are reflected in the indexes + + +@shared_task +def refresh_rfc_index_task(): + if rfcindex_is_dirty(): + # new_processed_time is the *start* of processing so that any changes after + # this point will trigger another refresh + new_processed_time = timezone.now() + + try: + create_rfc_txt_index() + except Exception as e: + log.log(f"Error: failure in creating rfc-index.txt. {e}") + pass + + try: + create_rfc_xml_index() + except Exception as e: + log.log(f"Error: failure in creating rfc-index.xml. {e}") + pass + + try: + create_bcp_txt_index() + except Exception as e: + log.log(f"Error: failure in creating bcp-index.txt. {e}") + pass + + try: + create_std_txt_index() + except Exception as e: + log.log(f"Error: failure in creating std-index.txt. {e}") + pass + + try: + create_fyi_txt_index() + except Exception as e: + log.log(f"Error: failure in creating fyi-index.txt. {e}") + pass + + mark_rfcindex_as_processed(new_processed_time) + + +@shared_task +def process_rpc_queue_task(data: list): + in_progress_state = State.objects.get( + used=True, type="draft-rfceditor", slug="in_progress" + ) + blocked_state = State.objects.get(used=True, type="draft-rfceditor", slug="blocked") + system = Person.objects.get(name="(System)") + iana_ref_tags = list(DocTagName.objects.filter(slug__in=["iana", "ref"])) + + names = [obj["name"] for obj in data] + docs_in_db = { + d.name: d for d in Document.objects.filter(type="draft", name__in=names) + } + + for obj in data: + name = obj["name"] + if name not in docs_in_db: + log.log(f"process_rpc_queue_task: unknown document {name}") + continue + + d = docs_in_db[name] + events = [] + prev_state = d.get_state("draft-rfceditor") + + # Same check as ietf.sync.rfceditor.update_drafts_from_queue: + # if this document just arrived at the RFC Editor for the first time, record it. + if ( + d.get_state_slug("draft-iesg") == "ann" + and not prev_state + and not d.latest_event(DocEvent, type="rfc_editor_received_announcement") + ): + e = DocEvent( + doc=d, rev=d.rev, by=system, type="rfc_editor_received_announcement" + ) + e.desc = "Announcement was received by RFC Editor" + e.save() + send_mail_text( + None, + "iesg-secretary@ietf.org", + None, + "%s in RFC Editor queue" % d.name, + "The announcement for %s has been received by the RFC Editor." % d.name, + ) + prev_iesg_state = State.objects.get( + used=True, type="draft-iesg", slug="ann" + ) + next_iesg_state = State.objects.get( + used=True, type="draft-iesg", slug="rfcqueue" + ) + d.set_state(next_iesg_state) + e = add_state_change_event(d, system, prev_iesg_state, next_iesg_state) + if e: + events.append(e) + e = update_action_holders(d, prev_iesg_state, next_iesg_state) + if e: + events.append(e) + + is_blocked = any(a["role"] == "blocked" for a in obj.get("assignment_set", [])) + next_state = blocked_state if is_blocked else in_progress_state + + if prev_state != next_state: + d.set_state(next_state) + e = new_state_change_event(d, system, prev_state, next_state) + if e: + e.save() + events.append(e) + + roles = sorted(a["role"] for a in obj.get("assignment_set", [])) + next_assignments = ", ".join(roles) + blocking_names = sorted( + br["reason"]["name"] for br in obj.get("blocking_reasons", []) + ) + if blocking_names: + next_assignments += ": " + ", ".join(blocking_names) + + if next_assignments == "": + next_assignments = "Awaiting Editor Assignment" + + prev_assignments_event = d.latest_event( + RpcAssignmentDocEvent, type="changed_rpc_assignments" + ) + prev_assignments = ( + prev_assignments_event.assignments if prev_assignments_event else None + ) + + if next_assignments != prev_assignments: + e = RpcAssignmentDocEvent( + doc=d, + rev=d.rev, + by=system, + type="changed_rpc_assignments", + assignments=next_assignments, + ) + e.desc = f"RPC status changed to {next_assignments}" + if prev_assignments is not None and prev_assignments != "": + e.desc += f" from {prev_assignments}" + e.save() + events.append(e) + + rfc_number = obj.get("rfc_number") + if obj.get("final_approval") and rfc_number: + d.documenturl_set.update_or_create( + tag_id="auth48", + defaults=dict( + url=f"{settings.RFC_EDITOR_QUEUE_SITE_BASE_URL}/final-review/rfc{rfc_number}/" + ), + ) + else: + d.documenturl_set.filter(tag_id="auth48").delete() + + d.tags.remove(*iana_ref_tags) + + if events: + d.save_with_history(events) + + for d in ( + Document.objects.exclude(name__in=names) + .filter(states__type="draft-rfceditor") + .distinct() + ): + d.tags.remove(*iana_ref_tags) + d.unset_state("draft-rfceditor") diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py index b0cdf863f0..207c78cf6a 100644 --- a/ietf/sync/tests.py +++ b/ietf/sync/tests.py @@ -1,18 +1,19 @@ -# Copyright The IETF Trust 2012-2020, All Rights Reserved -# -*- coding: utf-8 -*- +# Copyright The IETF Trust 2012-2026, All Rights Reserved import os import io import json import datetime -import mock +from unittest import mock import quopri import requests from dataclasses import dataclass from django.conf import settings +from django.core.files.base import ContentFile +from django.core.files.storage import storages from django.urls import reverse as urlreverse from django.utils import timezone from django.test.utils import override_settings @@ -20,14 +21,39 @@ import debug # pyflakes:ignore from ietf.api.views import EmailIngestionError -from ietf.doc.factories import WgDraftFactory, RfcFactory, DocumentAuthorFactory, DocEventFactory -from ietf.doc.models import Document, DocEvent, DeletedEvent, DocTagName, RelatedDocument, State, StateDocEvent +from ietf.doc.factories import ( + WgDraftFactory, + RfcFactory, + DocumentAuthorFactory, + DocEventFactory, + BcpFactory, WgRfcFactory, +) +from ietf.doc.models import ( + Document, + DocEvent, + DeletedEvent, + DocTagName, + RelatedDocument, + State, + StateDocEvent, +) from ietf.doc.utils import add_state_change_event from ietf.group.factories import GroupFactory from ietf.person.factories import PersonFactory from ietf.person.models import Person from ietf.sync import iana, rfceditor, tasks +from ietf.sync.errata import ( + update_errata_from_rfceditor, + get_errata_last_updated, + get_errata_data, + errata_map_from_json, + update_errata_dirty_time, + mark_errata_as_processed, + update_errata_tags, +) +from ietf.sync.tasks import update_errata_from_rfceditor_task from ietf.utils.mail import outbox, empty_outbox +from ietf.utils.models import DirtyBits from ietf.utils.test_utils import login_testing_unauthorized from ietf.utils.test_utils import TestCase from ietf.utils.timezone import date_today, RPC_TZINFO @@ -200,7 +226,7 @@ def test_iana_review_mail(self): doc_name, review_time, by, comment = iana.parse_review_email(msg.encode('utf-8')) self.assertEqual(doc_name, draft.name) - self.assertEqual(review_time, datetime.datetime(2012, 5, 10, 12, 0, rtime, tzinfo=datetime.timezone.utc)) + self.assertEqual(review_time, datetime.datetime(2012, 5, 10, 12, 0, rtime, tzinfo=datetime.UTC)) self.assertEqual(by, Person.objects.get(user__username="iana")) self.assertIn("there are no IANA Actions", comment.replace("\n", "")) @@ -234,7 +260,7 @@ def test_ingest_review_email(self, mock_parse_review_email, mock_add_review_comm args = ( "doc-name", - datetime.datetime.now(tz=datetime.timezone.utc), + datetime.datetime.now(tz=datetime.UTC), PersonFactory(), "yadda yadda yadda", ) @@ -295,6 +321,7 @@ def test_rfc_index(self): ad=Person.objects.get(user__username='ad'), external_url="http://my-external-url.example.com", note="this is a note", + pages=54, # make sure this is not 42 ) DocumentAuthorFactory.create_batch(2, document=draft_doc) draft_doc.action_holders.add(draft_doc.ad) # not normally set, but add to be sure it's cleared @@ -440,7 +467,7 @@ def test_rfc_index(self): rfc_doc = Document.objects.filter(rfc_number=1234, type_id="rfc").first() self.assertIsNotNone(rfc_doc, "RFC document should have been created") - self.assertEqual(rfc_doc.authors(), draft_doc.authors()) + self.assertEqual(rfc_doc.author_persons_or_names(), draft_doc.author_persons_or_names()) rfc_events = rfc_doc.docevent_set.all() self.assertEqual(len(rfc_events), 8) expected_events = [ @@ -508,6 +535,120 @@ def test_rfc_index(self): changed = list(rfceditor.update_docs_from_rfc_index(data, errata, today - datetime.timedelta(days=30))) self.assertEqual(len(changed), 0) + def test_rfc_index_subseries_replacement(self): + today = date_today() + author = PersonFactory(name="Some Bozo") + + # Start with two BCPs, each containing an rfc + rfc1, rfc2, rfc3 = RfcFactory.create_batch(3, authors=[author]) + bcp1 = BcpFactory(contains=[rfc1]) + bcp2 = BcpFactory(contains=[rfc2]) + + def _nameify(doc): + """Convert a name like 'rfc1' to 'RFC0001""" + return f"{doc.name[:3].upper()}{int(doc.name[3:]):04d}" + + # RFC index that replaces rfc2 with rfc3 in bcp2 + index_xml = f""" + + + {_nameify(bcp1)} + + {_nameify(rfc1)} + + + + {_nameify(bcp2)} + + {_nameify(rfc3)} + + + + {_nameify(rfc1)} + {rfc1.title} + + Some Bozo + + + {today.strftime('%B')} + {today.strftime('%Y')} + + + ASCII + + 42 + + test + +

    This is some interesting text.

     + + {_nameify(bcp1)} + + PROPOSED STANDARD + PROPOSED STANDARD + IETF +     + + {_nameify(rfc2)} + {rfc2.title} + + Some Bozo + + + {today.strftime('%B')} + {today.strftime('%Y')} + + + ASCII + + 42 + + test + +

    This is some interesting text.

     + PROPOSED STANDARD + PROPOSED STANDARD + IETF +     + + {_nameify(rfc3)} + {rfc3.title} + + Some Bozo + + + {today.strftime('%B')} + {today.strftime('%Y')} + + + ASCII + + 42 + + test + +

    This is some interesting text.

     + + {_nameify(bcp2)} + + PROPOSED STANDARD + PROPOSED STANDARD + IETF +     +    """ + data = rfceditor.parse_index(io.StringIO(index_xml)) # parse index + self.assertEqual(len(data), 3) # check that we parsed 3 RFCs + # Process the data by consuming the generator + for _ in rfceditor.update_docs_from_rfc_index(data, []): + pass + # Confirm that the expected changes were made + self.assertCountEqual(rfc1.related_that("contains"), [bcp1]) + self.assertCountEqual(rfc2.related_that("contains"), []) + self.assertCountEqual(rfc3.related_that("contains"), [bcp2]) + def _generate_rfc_queue_xml(self, draft, state, auth48_url=None): """Generate an RFC queue xml string for a draft""" t = ''' @@ -654,30 +795,6 @@ def test_update_draft_auth48_url(self): auth48_docurl = draft.documenturl_set.filter(tag_id='auth48').first() self.assertIsNone(auth48_docurl) - def test_post_approved_draft_in_production_only(self): - self.requests_mock.post("https://rfceditor.example.com/", status_code=200, text="OK") - - # be careful playing with SERVER_MODE! - with override_settings(SERVER_MODE="test"): - self.assertEqual( - rfceditor.post_approved_draft("https://rfceditor.example.com/", "some-draft"), - ("", "") - ) - self.assertFalse(self.requests_mock.called) - with override_settings(SERVER_MODE="development"): - self.assertEqual( - rfceditor.post_approved_draft("https://rfceditor.example.com/", "some-draft"), - ("", "") - ) - self.assertFalse(self.requests_mock.called) - with override_settings(SERVER_MODE="production"): - self.assertEqual( - rfceditor.post_approved_draft("https://rfceditor.example.com/", "some-draft"), - ("", "") - ) - self.assertTrue(self.requests_mock.called) - - class DiscrepanciesTests(TestCase): def test_discrepancies(self): @@ -761,6 +878,191 @@ def test_rfceditor_undo(self): self.assertTrue(StateDocEvent.objects.filter(desc="First", doc=draft)) +class ErrataTests(TestCase): + @override_settings(ERRATA_JSON_BLOB_NAME="myblob.json") + def test_get_errata_last_update(self): + red_bucket = storages["red_bucket"] # InMemoryStorage in test + red_bucket.save("myblob.json", ContentFile("file")) + self.assertEqual( + get_errata_last_updated(), red_bucket.get_modified_time("myblob.json") + ) + + @override_settings(ERRATA_JSON_BLOB_NAME="myblob.json") + def test_get_errata_data(self): + red_bucket = storages["red_bucket"] # InMemoryStorage in test + red_bucket.save("myblob.json", ContentFile('[{"value": 3}]')) + self.assertEqual( + get_errata_data(), + [{"value": 3}], + ) + + def test_errata_map_from_json(self): + input_data = [ + { + "doc-id": "not-an-rfc", + "errata_status_code": "Verified", + }, + { + "doc-id": "rfc01234", + "errata_status_code": "Reported", + }, + { + "doc-id": "RFC1001", + "errata_status_code": "Verified" + }, + { + "doc-id": "RfC1234", + "errata_status_code": "Verified", + }, + ] + expected_output = {1001: [input_data[2]], 1234: [input_data[1], input_data[3]]} + self.assertDictEqual(errata_map_from_json(input_data), expected_output) + + @mock.patch("ietf.sync.errata.update_errata_tags") + @mock.patch("ietf.sync.errata.get_errata_data") + def test_update_errata_from_rfceditor(self, mock_get_data, mock_update): + fake_data = object() + mock_get_data.return_value = fake_data + update_errata_from_rfceditor() + self.assertTrue(mock_get_data.called) + self.assertTrue(mock_update.called) + self.assertEqual(mock_update.call_args, mock.call(fake_data)) + + def test_update_errata_tags(self): + tag_has_errata = DocTagName.objects.get(slug="errata") + tag_has_verified_errata = DocTagName.objects.get(slug="verified-errata") + + rfcs = WgRfcFactory.create_batch(10) + rfcs[0].tags.set([tag_has_errata]) + rfcs[1].tags.set([tag_has_errata, tag_has_verified_errata]) + rfcs[2].tags.set([tag_has_errata]) + rfcs[3].tags.set([tag_has_errata, tag_has_verified_errata]) + rfcs[4].tags.set([tag_has_errata]) + rfcs[5].tags.set([tag_has_errata, tag_has_verified_errata]) + + # Only contains the fields we care about, not the full JSON + errata_data = [ + # rfcs[0] had errata and should keep it + {"doc-id": rfcs[0].name, "errata_status_code": "Held for Document Update"}, + {"doc-id": rfcs[0].name, "errata_status_code": "Rejected"}, + # rfcs[1] had errata+verified-errata and should keep both + {"doc-id": rfcs[1].name, "errata_status_code": "Verified"}, + # rfcs[2] had errata and should gain verified-errata + {"doc-id": rfcs[2].name, "errata_status_code": "Verified"}, + # rfcs[3] had errata+verified errata and should lose both + {"doc-id": rfcs[3].name, "errata_status_code": "Rejected"}, + # rfcs[4] had errata and should gain verified-errata + {"doc-id": rfcs[4].name, "errata_status_code": "Verified"}, + {"doc-id": rfcs[4].name, "errata_status_code": "Reported"}, + # rfcs[5] had errata+verified-errata and should lose verified-errata + {"doc-id": rfcs[5].name, "errata_status_code": "Reported"}, + # rfcs[6] had none and should gain errata + {"doc-id": rfcs[6].name, "errata_status_code": "Reported"}, + # rfcs[7] had none and should gain errata+verified-errata + {"doc-id": rfcs[7].name, "errata_status_code": "Verified"}, + # rfcs[8] had none and it should stay that way + {"doc-id": rfcs[8].name, "errata_status_code": "Rejected"}, + # rfcs[9] had none and it should stay that way (no entry at all) + ] + update_errata_tags(errata_data) + + self.assertCountEqual(rfcs[0].tags.all(), [tag_has_errata]) + self.assertIsNone(rfcs[0].docevent_set.first()) # no change + + self.assertCountEqual( + rfcs[1].tags.all(), [tag_has_errata, tag_has_verified_errata] + ) + self.assertIsNone(rfcs[1].docevent_set.first()) # no change + + self.assertCountEqual( + rfcs[2].tags.all(), [tag_has_errata, tag_has_verified_errata] + ) + self.assertEqual(rfcs[2].docevent_set.count(), 1) + self.assertIn(": added verified-errata tag", rfcs[2].docevent_set.first().desc) + + self.assertCountEqual(rfcs[3].tags.all(), []) + self.assertEqual(rfcs[3].docevent_set.count(), 1) + self.assertIn( + ": removed errata tag, removed verified-errata tag (all errata rejected)", + rfcs[3].docevent_set.first().desc, + ) + + self.assertCountEqual( + rfcs[4].tags.all(), [tag_has_errata, tag_has_verified_errata] + ) + self.assertEqual(rfcs[4].docevent_set.count(), 1) + self.assertIn(": added verified-errata tag", rfcs[4].docevent_set.first().desc) + + self.assertCountEqual(rfcs[5].tags.all(), [tag_has_errata]) + self.assertEqual(rfcs[5].docevent_set.count(), 1) + self.assertIn( + ": removed verified-errata tag", rfcs[5].docevent_set.first().desc + ) + + self.assertCountEqual(rfcs[6].tags.all(), [tag_has_errata]) + self.assertEqual(rfcs[6].docevent_set.count(), 1) + self.assertIn(": added errata tag", rfcs[6].docevent_set.first().desc) + + self.assertCountEqual( + rfcs[7].tags.all(), [tag_has_errata, tag_has_verified_errata] + ) + self.assertEqual(rfcs[7].docevent_set.count(), 1) + self.assertIn( + ": added errata tag, added verified-errata tag", + rfcs[7].docevent_set.first().desc, + ) + + self.assertCountEqual(rfcs[8].tags.all(), []) + self.assertIsNone(rfcs[8].docevent_set.first()) # no change + + self.assertCountEqual(rfcs[9].tags.all(), []) + self.assertIsNone(rfcs[9].docevent_set.first()) # no change + + @override_settings(ERRATA_JSON_BLOB_NAME="myblob.json") + @mock.patch("ietf.sync.errata.get_errata_last_updated") + def test_update_errata_dirty_time(self, mock_last_updated): + ERRATA_SLUG = DirtyBits.Slugs.ERRATA + + # No time available + mock_last_updated.side_effect = FileNotFoundError + self.assertIsNone(DirtyBits.objects.filter(slug=ERRATA_SLUG).first()) + self.assertIsNone(update_errata_dirty_time()) # no blob yet + self.assertIsNone(DirtyBits.objects.filter(slug=ERRATA_SLUG).first()) + + # Now set a time + first_timestamp = timezone.now() - datetime.timedelta(hours=3) + mock_last_updated.return_value = first_timestamp + mock_last_updated.side_effect = None + result = update_errata_dirty_time() + self.assertTrue(isinstance(result, DirtyBits)) + result.refresh_from_db() + self.assertEqual(result.slug, ERRATA_SLUG) + self.assertEqual(result.processed_time, None) + self.assertEqual(result.dirty_time, first_timestamp) + + # Update the time + second_timestamp = timezone.now() + mock_last_updated.return_value = second_timestamp + second_result = update_errata_dirty_time() + self.assertEqual(result.pk, second_result.pk) # should be the same record + result.refresh_from_db() + self.assertEqual(result.slug, ERRATA_SLUG) + self.assertEqual(result.processed_time, None) + self.assertEqual(result.dirty_time, second_timestamp) + + def test_mark_errata_as_processed(self): + ERRATA_SLUG = DirtyBits.Slugs.ERRATA + first_timestamp = timezone.now() + mark_errata_as_processed(first_timestamp) # no DirtyBits is not an error + self.assertIsNone(DirtyBits.objects.filter(slug=ERRATA_SLUG).first()) + dbits = DirtyBits.objects.create(slug=ERRATA_SLUG, dirty_time=first_timestamp) + second_timestamp = timezone.now() + mark_errata_as_processed(second_timestamp) + dbits.refresh_from_db() + self.assertEqual(dbits.dirty_time, first_timestamp) + self.assertEqual(dbits.processed_time, second_timestamp) + + class TaskTests(TestCase): @override_settings( RFC_EDITOR_INDEX_URL="https://rfc-editor.example.com/index/", @@ -769,8 +1071,9 @@ class TaskTests(TestCase): @mock.patch("ietf.sync.tasks.rfceditor.update_docs_from_rfc_index") @mock.patch("ietf.sync.tasks.rfceditor.parse_index") @mock.patch("ietf.sync.tasks.requests.get") + @mock.patch("ietf.sync.tasks.rsync_rfcs_from_rfceditor_task.delay") def test_rfc_editor_index_update_task( - self, requests_get_mock, parse_index_mock, update_docs_mock + self, rsync_task_mock, requests_get_mock, parse_index_mock, update_docs_mock ) -> None: # the annotation here prevents mypy from complaining about annotation-unchecked """rfc_editor_index_update_task calls helpers correctly @@ -802,6 +1105,7 @@ def json(self): rfc = RfcFactory() # Test with full_index = False + rsync_task_mock.return_value = None requests_get_mock.side_effect = (index_response, errata_response) # will step through these parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS) update_docs_mock.return_value = ( @@ -827,10 +1131,13 @@ def json(self): ) self.assertIsNotNone(update_docs_kwargs["skip_older_than_date"]) + self.assertFalse(rsync_task_mock.called) + # Test again with full_index = True requests_get_mock.reset_mock() parse_index_mock.reset_mock() update_docs_mock.reset_mock() + rsync_task_mock.reset_mock() requests_get_mock.side_effect = (index_response, errata_response) # will step through these tasks.rfc_editor_index_update_task(full_index=True) @@ -851,40 +1158,67 @@ def json(self): ) self.assertIsNone(update_docs_kwargs["skip_older_than_date"]) + self.assertFalse(rsync_task_mock.called) + + # Test again where the index would cause a new RFC to come into existance + requests_get_mock.reset_mock() + parse_index_mock.reset_mock() + update_docs_mock.reset_mock() + rsync_task_mock.reset_mock() + requests_get_mock.side_effect = ( + index_response, + errata_response, + ) # will step through these + update_docs_mock.return_value = ( + (rfc.rfc_number, ("something changed",), rfc, True), + ) + tasks.rfc_editor_index_update_task(full_index=True) + self.assertTrue(rsync_task_mock.called) + rsync_task_args, rsync_task_kwargs = rsync_task_mock.call_args + self.assertEqual((([rfc.rfc_number],), {}), (rsync_task_args, rsync_task_kwargs)) + # Test error handling requests_get_mock.reset_mock() parse_index_mock.reset_mock() update_docs_mock.reset_mock() + rsync_task_mock.reset_mock() requests_get_mock.side_effect = requests.Timeout # timeout on every get() tasks.rfc_editor_index_update_task(full_index=False) self.assertFalse(parse_index_mock.called) self.assertFalse(update_docs_mock.called) + self.assertFalse(rsync_task_mock.called) requests_get_mock.reset_mock() parse_index_mock.reset_mock() update_docs_mock.reset_mock() + rsync_task_mock.reset_mock() requests_get_mock.side_effect = [index_response, requests.Timeout] # timeout second get() tasks.rfc_editor_index_update_task(full_index=False) self.assertFalse(update_docs_mock.called) + self.assertFalse(rsync_task_mock.called) requests_get_mock.reset_mock() parse_index_mock.reset_mock() update_docs_mock.reset_mock() + rsync_task_mock.reset_mock() requests_get_mock.side_effect = [index_response, errata_response] # feed in an index that is too short parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS - 1) tasks.rfc_editor_index_update_task(full_index=False) self.assertTrue(parse_index_mock.called) self.assertFalse(update_docs_mock.called) + self.assertFalse(rsync_task_mock.called) requests_get_mock.reset_mock() parse_index_mock.reset_mock() update_docs_mock.reset_mock() + rsync_task_mock.reset_mock() requests_get_mock.side_effect = [index_response, errata_response] errata_response.json_length = rfceditor.MIN_ERRATA_RESULTS - 1 # too short parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS) tasks.rfc_editor_index_update_task(full_index=False) self.assertFalse(update_docs_mock.called) + self.assertFalse(rsync_task_mock.called) @override_settings(RFC_EDITOR_QUEUE_URL="https://rfc-editor.example.com/queue/") @mock.patch("ietf.sync.tasks.update_drafts_from_queue") @@ -1001,7 +1335,7 @@ def test_iana_protocols_update_task( ) self.assertEqual( published_later_than, - {datetime.datetime(2012,11,26,tzinfo=datetime.timezone.utc)} + {datetime.datetime(2012,11,26,tzinfo=datetime.UTC)} ) # try with an exception @@ -1014,3 +1348,76 @@ def test_iana_protocols_update_task( self.assertTrue(requests_get_mock.called) self.assertFalse(parse_protocols_mock.called) self.assertFalse(update_rfc_log_mock.called) + + @mock.patch("ietf.sync.tasks.rsync_helper") + @mock.patch("ietf.sync.tasks.load_rfcs_into_blobdb") + @mock.patch("ietf.sync.tasks.rebuild_reference_relations_task.delay") + def test_rsync_rfcs_from_rfceditor_task( + self, + rebuild_relations_mock, + load_blobs_mock, + rsync_helper_mock, + ): + tasks.rsync_rfcs_from_rfceditor_task([12345, 54321]) + self.assertTrue(rsync_helper_mock.called) + self.assertTrue(load_blobs_mock.called) + load_blobs_args, load_blobs_kwargs = load_blobs_mock.call_args + self.assertEqual(load_blobs_args, ([12345, 54321],)) + self.assertEqual(load_blobs_kwargs, {}) + self.assertTrue(rebuild_relations_mock.called) + rebuild_args, rebuild_kwargs = rebuild_relations_mock.call_args + self.assertEqual(rebuild_args, (["rfc12345", "rfc54321"],)) + self.assertEqual(rebuild_kwargs, {}) + + @mock.patch("ietf.sync.tasks.load_rfcs_into_blobdb") + def test_load_rfcs_into_blobdb_task( + self, + load_blobs_mock, + ): + tasks.load_rfcs_into_blobdb_task(5, 3) + self.assertFalse(load_blobs_mock.called) + load_blobs_mock.reset_mock() + tasks.load_rfcs_into_blobdb_task(-1, 1) + self.assertTrue(load_blobs_mock.called) + mock_args, mock_kwargs = load_blobs_mock.call_args + self.assertEqual(mock_args, ([1],)) + self.assertEqual(mock_kwargs, {}) + load_blobs_mock.reset_mock() + tasks.load_rfcs_into_blobdb_task(10999, 50000) + self.assertTrue(load_blobs_mock.called) + mock_args, mock_kwargs = load_blobs_mock.call_args + self.assertEqual(mock_args, ([10999, 11000],)) + self.assertEqual(mock_kwargs, {}) + load_blobs_mock.reset_mock() + tasks.load_rfcs_into_blobdb_task(3261, 3263) + self.assertTrue(load_blobs_mock.called) + mock_args, mock_kwargs = load_blobs_mock.call_args + self.assertEqual(mock_args, ([3261, 3262, 3263],)) + self.assertEqual(mock_kwargs, {}) + + + @mock.patch("ietf.sync.tasks.update_errata_from_rfceditor") + @mock.patch("ietf.sync.tasks.mark_rfcindex_as_dirty") + @mock.patch("ietf.sync.tasks.mark_errata_as_processed") + @mock.patch("ietf.sync.tasks.errata_are_dirty") + def test_update_errata_from_rfceditor_task( + self, + mock_errata_are_dirty, + mock_mark_errata_processed, + mock_mark_rfcindex_dirty, + mock_update, + ): + mock_errata_are_dirty.return_value = False + update_errata_from_rfceditor_task() + self.assertTrue(mock_errata_are_dirty.called) + self.assertFalse(mock_mark_errata_processed.called) + self.assertFalse(mock_mark_rfcindex_dirty.called) + self.assertFalse(mock_update.called) + + mock_errata_are_dirty.reset_mock() + mock_errata_are_dirty.return_value = True + update_errata_from_rfceditor_task() + self.assertTrue(mock_errata_are_dirty.called) + self.assertTrue(mock_mark_errata_processed.called) + self.assertTrue(mock_mark_rfcindex_dirty.called) + self.assertTrue(mock_update.called) diff --git a/ietf/sync/tests_rfcindex.py b/ietf/sync/tests_rfcindex.py new file mode 100644 index 0000000000..2b70924db3 --- /dev/null +++ b/ietf/sync/tests_rfcindex.py @@ -0,0 +1,473 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +import json +import re +from pathlib import Path +from unittest import mock + +from django.conf import settings +from django.core.files.base import ContentFile +from django.core.files.storage import storages +from django.test.utils import override_settings +from lxml import etree + +from ietf.doc.factories import ( + BcpFactory, + FyiFactory, + StdFactory, + IndividualRfcFactory, + PublishedRfcDocEventFactory, +) +from ietf.name.models import DocTagName +from ietf.sync.rfcindex import ( + create_bcp_txt_index, + create_fyi_txt_index, + create_rfc_txt_index, + create_rfc_xml_index, + create_std_txt_index, + format_rfc_number, + get_april1_rfc_numbers, + get_publication_std_levels, + get_unusable_rfc_numbers, + save_to_red_bucket, + subseries_text_line, + save_to_filesystem, +) +from ietf.utils.test_utils import TestCase + + +class RfcIndexTests(TestCase): + """Tests of rfc-index generation + + Tests are limited and should cover more cases. Needs: + * test of subseries docs + * test of related docs (obsoletes/updates + reverse directions) + * more thorough validation of index contents + + Be careful when calling create_rfc_txt_index() or create_rfc_xml_index(). These + will save to a storage by default, which can introduce cross-talk between tests. + Best to patch that method with a mock. + """ + + def setUp(self): + super().setUp() + red_bucket = storages["red_bucket"] + + # Create an unused RFC number + red_bucket.save( + "input/unusable-rfc-numbers.json", + ContentFile(json.dumps([{"number": 123, "comment": ""}])), + ) + + # actual April 1 RFC + self.april_fools_rfc = PublishedRfcDocEventFactory( + time="2020-04-01T12:00:00Z", + doc=IndividualRfcFactory( + name="rfc4560", + rfc_number=4560, + stream_id="ise", + std_level_id="inf", + ), + ).doc + # Set up a JSON file to flag the April 1 RFC + red_bucket.save( + "input/april-first-rfc-numbers.json", + ContentFile(json.dumps([self.april_fools_rfc.rfc_number])), + ) + + # non-April Fools RFC that happens to have been published on April 1 + self.rfc = PublishedRfcDocEventFactory( + time="2021-04-01T12:00:00Z", + doc__name="rfc10000", + doc__rfc_number=10000, + doc__std_level_id="std", + ).doc + self.rfc.tags.add(DocTagName.objects.get(slug="errata")) + + # Create a BCP with non-April Fools RFC + self.bcp = BcpFactory(contains=[self.rfc], name="bcp11") + + # Create a STD with non-April Fools RFC + self.std = StdFactory(contains=[self.rfc], name="std11") + + # Create a FYI with non-April Fools RFC + self.fyi = FyiFactory(contains=[self.rfc], name="fyi11") + + # Set up a publication-std-levels.json file to indicate the publication + # standard of self.rfc as different from its current value + red_bucket.save( + "input/publication-std-levels.json", + ContentFile( + json.dumps( + [{"number": self.rfc.rfc_number, "publication_std_level": "ps"}] + ) + ), + ) + + def tearDown(self): + red_bucket = storages["red_bucket"] + red_bucket.delete("input/unusable-rfc-numbers.json") + red_bucket.delete("input/april-first-rfc-numbers.json") + red_bucket.delete("input/publication-std-levels.json") + super().tearDown() + + @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_filesystem") + @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") + def test_create_rfc_txt_index(self, mock_save_blob, mock_save_file): + create_rfc_txt_index() + self.assertEqual(mock_save_blob.call_count, 1) + self.assertEqual(mock_save_blob.call_args[0][0], "rfc-index.txt") + contents = mock_save_blob.call_args[0][1] + + self.assertEqual(mock_save_file.call_count, 1) + self.assertEqual(mock_save_file.call_args, mock.call("rfc-index.txt", contents)) + + self.assertTrue(isinstance(contents, str)) + self.assertIn( + "123 Not Issued.", + contents, + ) + # No zero prefix! + self.assertNotIn( + "0123 Not Issued.", + contents, + ) + + # strip whitespace so line breaks don't interfere with the next few tests + stripped_contents = re.sub(r"\s+", " ", mock_save_blob.call_args[0][1]) + self.assertIn( + f"{self.april_fools_rfc.rfc_number} {self.april_fools_rfc.title}", + stripped_contents, + ) + self.assertIn("1 April 2020", contents) # from the April 1 RFC + self.assertIn( + f"{self.rfc.rfc_number} {self.rfc.title}", + stripped_contents, + ) + self.assertIn("April 2021", stripped_contents) # from the non-April 1 RFC + self.assertNotIn("1 April 2021", stripped_contents) + + @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_filesystem") + @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") + def test_create_rfc_xml_index(self, mock_save_blob, mock_save_file): + create_rfc_xml_index() + self.assertEqual(mock_save_blob.call_count, 1) + self.assertEqual(mock_save_blob.call_args[0][0], "rfc-index.xml") + contents = mock_save_blob.call_args[0][1] + + self.assertEqual(mock_save_file.call_count, 1) + self.assertEqual(mock_save_file.call_args, mock.call("rfc-index.xml", contents)) + + self.assertTrue(isinstance(contents, bytes)) + ns = "{https://www.rfc-editor.org/rfc-index}" # NOT an f-string + index = etree.fromstring(contents) + + # We can aspire to validating the schema - currently does not conform because + # XSD expects 4-digit RFC numbers (etc). + # + # xmlschema = etree.XMLSchema(etree.fromstring( + # Path(__file__).with_name("rfc-index.xsd").read_bytes()) + # ) + # xmlschema.assertValid(index) + + children = list(index) # elements as list + # Should be one rfc-not-issued-entry + self.assertEqual(len(children), 16) + self.assertEqual( + [ + c.find(f"{ns}doc-id").text + for c in children + if c.tag == f"{ns}rfc-not-issued-entry" + ], + ["RFC123"], + ) + # Should be two rfc-entries + rfc_entries = { + c.find(f"{ns}doc-id").text: c for c in children if c.tag == f"{ns}rfc-entry" + } + + # Check the April Fool's entry + april_fools_entry = rfc_entries[self.april_fools_rfc.name.upper()] + self.assertEqual( + april_fools_entry.find(f"{ns}title").text, + self.april_fools_rfc.title, + ) + self.assertEqual( + [(c.tag, c.text) for c in april_fools_entry.find(f"{ns}date")], + [(f"{ns}month", "April"), (f"{ns}day", "1"), (f"{ns}year", "2020")], + ) + self.assertEqual( + april_fools_entry.find(f"{ns}current-status").text, + "INFORMATIONAL", + ) + self.assertEqual( + april_fools_entry.find(f"{ns}publication-status").text, + "UNKNOWN", + ) + + # Check the Regular entry + rfc_entry = rfc_entries[self.rfc.name.upper()] + self.assertEqual(rfc_entry.find(f"{ns}title").text, self.rfc.title) + self.assertEqual( + rfc_entry.find(f"{ns}current-status").text, "INTERNET STANDARD" + ) + self.assertEqual( + rfc_entry.find(f"{ns}publication-status").text, "PROPOSED STANDARD" + ) + self.assertEqual( + [(c.tag, c.text) for c in rfc_entry.find(f"{ns}date")], + [(f"{ns}month", "April"), (f"{ns}year", "2021")], + ) + + @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_filesystem") + @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") + def test_create_bcp_txt_index(self, mock_save_blob, mock_save_file): + create_bcp_txt_index() + self.assertEqual(mock_save_blob.call_count, 1) + self.assertEqual(mock_save_blob.call_args[0][0], "bcp-index.txt") + contents = mock_save_blob.call_args[0][1] + + self.assertEqual(mock_save_file.call_count, 1) + self.assertEqual( + mock_save_file.call_args, + mock.call("bcp-index.txt", contents, ["bcp"]), + ) + + self.assertTrue(isinstance(contents, str)) + # starts from 1 + self.assertIn( + "[BCP1]", + contents, + ) + # fill up to 11 + self.assertIn( + "[BCP10]", + contents, + ) + # but not to 12 + self.assertNotIn( + "[BCP12]", + contents, + ) + # Test empty BCPs + self.assertIn( + "Best Current Practice 9 currently contains no RFCs", + contents, + ) + # No zero prefix! + self.assertNotIn( + "[BCP0001]", + contents, + ) + # Has BCP11 with a RFC + self.assertIn( + "Best Current Practice 11,", + contents, + ) + self.assertIn( + f'"{self.rfc.title}"', + contents, + ) + self.assertIn( + "BCP 11,", + contents, + ) + self.assertIn( + f"RFC {self.rfc.rfc_number},", + contents, + ) + + @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_filesystem") + @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") + def test_create_std_txt_index(self, mock_save_blob, mock_save_file): + create_std_txt_index() + self.assertEqual(mock_save_blob.call_count, 1) + self.assertEqual(mock_save_blob.call_args[0][0], "std-index.txt") + contents = mock_save_blob.call_args[0][1] + + self.assertEqual(mock_save_file.call_count, 1) + self.assertEqual( + mock_save_file.call_args, + mock.call("std-index.txt", contents, ["std"]), + ) + + self.assertTrue(isinstance(contents, str)) + # starts from 1 + self.assertIn( + "[STD1]", + contents, + ) + # fill up to 11 + self.assertIn( + "[STD10]", + contents, + ) + # but not to 12 + self.assertNotIn( + "[STD12]", + contents, + ) + # Test empty STDs + self.assertIn( + "Internet Standard 9 currently contains no RFCs", + contents, + ) + # No zero prefix! + self.assertNotIn( + "[STD0001]", + contents, + ) + # Has STD11 with a RFC + self.assertIn( + "Internet Standard 11,", + contents, + ) + self.assertIn( + f'"{self.rfc.title}"', + contents, + ) + self.assertIn( + "STD 11,", + contents, + ) + self.assertIn( + f"RFC {self.rfc.rfc_number},", + contents, + ) + + @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_filesystem") + @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") + def test_create_fyi_txt_index(self, mock_save_blob, mock_save_file): + create_fyi_txt_index() + self.assertEqual(mock_save_blob.call_count, 1) + self.assertEqual(mock_save_blob.call_args[0][0], "fyi-index.txt") + contents = mock_save_blob.call_args[0][1] + + self.assertEqual(mock_save_file.call_count, 1) + self.assertEqual( + mock_save_file.call_args, + mock.call("fyi-index.txt", contents, ["fyi"]), + ) + + self.assertTrue(isinstance(contents, str)) + # starts from 1 + self.assertIn( + "[FYI1]", + contents, + ) + # fill up to 11 + self.assertIn( + "[FYI10]", + contents, + ) + # but not to 12 + self.assertNotIn( + "[FYI12]", + contents, + ) + # Test empty FYIs + self.assertIn( + "For Your Information 9 currently contains no RFCs", + contents, + ) + # No zero prefix! + self.assertNotIn( + "[FYI0001]", + contents, + ) + # Has FYI11 with a RFC + self.assertIn( + "For Your Information 11,", + contents, + ) + self.assertIn( + f'"{self.rfc.title}"', + contents, + ) + self.assertIn( + "FYI 11,", + contents, + ) + self.assertIn( + f"RFC {self.rfc.rfc_number},", + contents, + ) + + +class HelperTests(TestCase): + def test_format_rfc_number(self): + self.assertEqual(format_rfc_number(10), "10") + with override_settings(RFCINDEX_MATCH_LEGACY_XML=True): + self.assertEqual(format_rfc_number(10), "0010") + + def test_save_to_red_bucket(self): + red_bucket = storages["red_bucket"] + with override_settings(RFCINDEX_DELETE_THEN_WRITE=False): + save_to_red_bucket("test", "contents \U0001f600") + # Read as binary and explicitly decode to confirm encoding + with red_bucket.open("test", "rb") as f: + self.assertEqual(f.read().decode("utf-8"), "contents \U0001f600") + with override_settings(RFCINDEX_DELETE_THEN_WRITE=True): + save_to_red_bucket("test", "new contents \U0001fae0".encode("utf-8")) + # Read as binary and explicitly decode to confirm encoding + with red_bucket.open("test", "rb") as f: + self.assertEqual(f.read().decode("utf-8"), "new contents \U0001fae0") + red_bucket.delete("test") # clean up like a good child + + def test_save_to_filesystem(self): + rfc_path = Path(settings.RFC_PATH) + self.assertFalse((rfc_path / "test").exists()) + save_to_filesystem("test", "contents \U0001f600") + self.assertEqual((rfc_path / "test").read_text("utf-8"), "contents \U0001f600") + self.assertFalse((rfc_path / "subdir" / "test").exists()) + + self.assertFalse((rfc_path / "test2").exists()) + self.assertFalse((rfc_path / "subdir" / "test2").exists()) + save_to_filesystem("test", "contents \U0001f600".encode("utf-8"), ["subdir"]) + self.assertEqual((rfc_path / "test").read_text("utf-8"), "contents \U0001f600") + self.assertEqual( + (rfc_path / "subdir" / "test").read_text("utf-8"), "contents \U0001f600" + ) + self.assertEqual( + (rfc_path / "test").stat().st_mtime, + (rfc_path / "subdir" / "test").stat().st_mtime, + ) + + def test_get_unusable_rfc_numbers_raises(self): + """get_unusable_rfc_numbers should bail on errors""" + with self.assertRaises(FileNotFoundError): + get_unusable_rfc_numbers() + red_bucket = storages["red_bucket"] + red_bucket.save("unusable-rfc-numbers.json", ContentFile("not json")) + with self.assertRaises(json.JSONDecodeError): + get_unusable_rfc_numbers() + red_bucket.delete("unusable-rfc-numbers.json") + + def test_get_april1_rfc_numbers_raises(self): + """get_april1_rfc_numbers should bail on errors""" + with self.assertRaises(FileNotFoundError): + get_april1_rfc_numbers() + red_bucket = storages["red_bucket"] + red_bucket.save("april-first-rfc-numbers.json", ContentFile("not json")) + with self.assertRaises(json.JSONDecodeError): + get_april1_rfc_numbers() + red_bucket.delete("april-first-rfc-numbers.json") + + def test_get_publication_std_levels_raises(self): + """get_publication_std_levels should bail on errors""" + with self.assertRaises(FileNotFoundError): + get_publication_std_levels() + red_bucket = storages["red_bucket"] + red_bucket.save("publication-std-levels.json", ContentFile("not json")) + with self.assertRaises(json.JSONDecodeError): + get_publication_std_levels() + red_bucket.delete("publication-std-levels.json") + + def test_subseries_text_line(self): + text = "foobar" + self.assertEqual(subseries_text_line(line=text, first=True), f" {text}") + self.assertEqual(subseries_text_line(line=text), f" {text}") diff --git a/ietf/sync/tests_tasks.py b/ietf/sync/tests_tasks.py new file mode 100644 index 0000000000..6657dd617a --- /dev/null +++ b/ietf/sync/tests_tasks.py @@ -0,0 +1,476 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.test.utils import override_settings + +from ietf.doc.factories import WgDraftFactory +from ietf.doc.models import ( + DocEvent, + DocTagName, + Document, + DocumentURL, + RpcAssignmentDocEvent, + State, +) +from ietf.person.models import Person +from ietf.sync import tasks +from ietf.utils.mail import outbox +from ietf.utils.test_utils import TestCase + + +def _make_entry( + doc_name, roles=None, blocking_reasons=None, rfc_number=None, final_approval=None +): + return { + "name": doc_name, + "assignment_set": [{"role": r, "state": "in_progress"} for r in (roles or [])], + "blocking_reasons": blocking_reasons or [], + "rfc_number": rfc_number, + "final_approval": final_approval or [], + } + + +class ProcessRpcQueueTaskTests(TestCase): + def setUp(self): + super().setUp() + self.system = Person.objects.get(name="(System)") + + # --- Unknown document -------------------------------------------------------- + + def test_unknown_document_is_skipped(self): + """Entries with unknown doc names are logged and skipped without raising.""" + tasks.process_rpc_queue_task([_make_entry("draft-does-not-exist")]) + self.assertFalse(Document.objects.filter(name="draft-does-not-exist").exists()) + + # --- First-arrival announcement ---------------------------------------------- + + def test_first_arrival_fires_announcement(self): + """Fires rfc_editor_received_announcement and email on first arrival.""" + draft = WgDraftFactory(states=[("draft-iesg", "ann")]) + mailbox_before = len(outbox) + + tasks.process_rpc_queue_task([_make_entry(draft.name, roles=["first_editor"])]) + + draft = Document.objects.get(pk=draft.pk) + self.assertEqual(draft.get_state_slug("draft-iesg"), "rfcqueue") + self.assertTrue( + draft.docevent_set.filter(type="rfc_editor_received_announcement").exists() + ) + self.assertEqual(len(outbox), mailbox_before + 1) + self.assertIn("RFC Editor queue", outbox[-1]["Subject"]) + self.assertIn("iesg-secretary@ietf.org", outbox[-1]["To"]) + + def test_first_arrival_skipped_if_rfceditor_state_exists(self): + """No announcement when doc already has a draft-rfceditor state.""" + draft = WgDraftFactory(states=[("draft-iesg", "ann")]) + draft.set_state( + State.objects.get(used=True, type="draft-rfceditor", slug="in_progress") + ) + mailbox_before = len(outbox) + + tasks.process_rpc_queue_task([_make_entry(draft.name, roles=["first_editor"])]) + + self.assertFalse( + draft.docevent_set.filter(type="rfc_editor_received_announcement").exists() + ) + self.assertEqual(len(outbox), mailbox_before) + + def test_first_arrival_skipped_if_announcement_event_exists(self): + """No duplicate announcement when rfc_editor_received_announcement already exists.""" + draft = WgDraftFactory(states=[("draft-iesg", "ann")]) + DocEvent.objects.create( + doc=draft, + rev=draft.rev, + by=self.system, + type="rfc_editor_received_announcement", + desc="Announcement was received by RFC Editor", + ) + mailbox_before = len(outbox) + + tasks.process_rpc_queue_task([_make_entry(draft.name)]) + + self.assertEqual( + draft.docevent_set.filter(type="rfc_editor_received_announcement").count(), + 1, + ) + self.assertEqual(len(outbox), mailbox_before) + + def test_first_arrival_skipped_if_not_ann_iesg_state(self): + """No announcement when IESG state is not 'ann'.""" + draft = WgDraftFactory(states=[("draft-iesg", "rfcqueue")]) + mailbox_before = len(outbox) + + tasks.process_rpc_queue_task([_make_entry(draft.name, roles=["first_editor"])]) + + self.assertFalse( + draft.docevent_set.filter(type="rfc_editor_received_announcement").exists() + ) + self.assertEqual(len(outbox), mailbox_before) + + # --- draft-rfceditor state transitions --------------------------------------- + + def test_sets_in_progress_state(self): + """Non-blocked assignment results in in_progress draft-rfceditor state.""" + draft = WgDraftFactory() + + tasks.process_rpc_queue_task([_make_entry(draft.name, roles=["first_editor"])]) + + draft = Document.objects.get(pk=draft.pk) + self.assertEqual(draft.get_state_slug("draft-rfceditor"), "in_progress") + + def test_sets_blocked_state(self): + """Assignment with role='blocked' results in blocked draft-rfceditor state.""" + draft = WgDraftFactory() + + tasks.process_rpc_queue_task( + [ + { + "name": draft.name, + "assignment_set": [{"role": "blocked", "state": "in_progress"}], + "blocking_reasons": [], + "rfc_number": None, + "final_approval": [], + } + ] + ) + + draft = Document.objects.get(pk=draft.pk) + self.assertEqual(draft.get_state_slug("draft-rfceditor"), "blocked") + + def test_no_state_change_event_when_state_unchanged(self): + """No state-change DocEvent created when draft-rfceditor state is already correct.""" + draft = WgDraftFactory(states=[("draft-rfceditor", "in_progress")]) + events_before = draft.docevent_set.filter(type="changed_state").count() + + tasks.process_rpc_queue_task([_make_entry(draft.name, roles=["first_editor"])]) + + self.assertEqual( + draft.docevent_set.filter(type="changed_state").count(), events_before + ) + + def test_state_change_event_created_on_transition(self): + """State-change DocEvent is created when draft-rfceditor state changes.""" + draft = WgDraftFactory(states=[("draft-rfceditor", "in_progress")]) + + tasks.process_rpc_queue_task( + [ + { + "name": draft.name, + "assignment_set": [{"role": "blocked", "state": "in_progress"}], + "blocking_reasons": [], + "rfc_number": None, + "final_approval": [], + } + ] + ) + + self.assertTrue(draft.docevent_set.filter(type="changed_state").exists()) + draft = Document.objects.get(pk=draft.pk) + self.assertEqual(draft.get_state_slug("draft-rfceditor"), "blocked") + + # --- RpcAssignmentDocEvent --------------------------------------------------- + + def test_creates_assignment_event_on_first_update(self): + """Creates RpcAssignmentDocEvent when no prior event exists.""" + draft = WgDraftFactory() + + tasks.process_rpc_queue_task( + [_make_entry(draft.name, roles=["first_editor", "second_editor"])] + ) + + event = draft.latest_event( + RpcAssignmentDocEvent, type="changed_rpc_assignments" + ) + self.assertIsNotNone(event) + self.assertEqual(event.assignments, "first_editor, second_editor") + + def test_no_assignment_event_when_unchanged(self): + """No new RpcAssignmentDocEvent when assignments match the last recorded ones.""" + draft = WgDraftFactory() + RpcAssignmentDocEvent.objects.create( + doc=draft, + rev=draft.rev, + by=self.system, + type="changed_rpc_assignments", + assignments="first_editor", + desc="RPC status changed to first_editor", + ) + events_before = RpcAssignmentDocEvent.objects.filter(doc=draft).count() + + tasks.process_rpc_queue_task([_make_entry(draft.name, roles=["first_editor"])]) + + self.assertEqual( + RpcAssignmentDocEvent.objects.filter(doc=draft).count(), events_before + ) + + def test_assignment_desc_includes_previous_assignments(self): + """Assignment event desc includes previous assignments when they exist.""" + draft = WgDraftFactory() + RpcAssignmentDocEvent.objects.create( + doc=draft, + rev=draft.rev, + by=self.system, + type="changed_rpc_assignments", + assignments="first_editor", + desc="RPC status changed to first_editor", + ) + + tasks.process_rpc_queue_task([_make_entry(draft.name, roles=["second_editor"])]) + + event = draft.latest_event( + RpcAssignmentDocEvent, type="changed_rpc_assignments" + ) + self.assertIn("from first_editor", event.desc) + + def test_blocking_reasons_appended_to_assignments(self): + """Blocking reason names are appended after ':' in the assignment string, sorted.""" + draft = WgDraftFactory() + + tasks.process_rpc_queue_task( + [ + { + "name": draft.name, + "assignment_set": [{"role": "blocked", "state": "in_progress"}], + "blocking_reasons": [ + {"reason": {"name": "missing reference"}}, + ], + "rfc_number": None, + "final_approval": [], + } + ] + ) + + event = draft.latest_event( + RpcAssignmentDocEvent, type="changed_rpc_assignments" + ) + self.assertIsNotNone(event) + self.assertEqual(event.assignments, "blocked: missing reference") + + def test_roles_sorted_in_assignment_string(self): + """Roles are sorted alphabetically in the assignment string.""" + draft = WgDraftFactory() + + tasks.process_rpc_queue_task( + [_make_entry(draft.name, roles=["second_editor", "first_editor"])] + ) + + event = draft.latest_event( + RpcAssignmentDocEvent, type="changed_rpc_assignments" + ) + self.assertEqual(event.assignments, "first_editor, second_editor") + + def test_empty_roles_uses_awaiting_editor_assignment(self): + """Empty assignment_set records 'Awaiting Editor Assignment' rather than an empty string.""" + draft = WgDraftFactory() + + tasks.process_rpc_queue_task([_make_entry(draft.name)]) + + event = draft.latest_event( + RpcAssignmentDocEvent, type="changed_rpc_assignments" + ) + self.assertIsNotNone(event) + self.assertEqual(event.assignments, "Awaiting Editor Assignment") + + # --- DocumentURL (auth48) handling ------------------------------------------- + + @override_settings(RFC_EDITOR_QUEUE_SITE_BASE_URL="https://queue.example.com") + def test_auth48_url_created_on_final_approval(self): + """auth48 DocumentURL is created when final_approval is truthy and rfc_number is set.""" + draft = WgDraftFactory() + + tasks.process_rpc_queue_task( + [ + { + "name": draft.name, + "assignment_set": [ + {"role": "first_editor", "state": "in_progress"} + ], + "blocking_reasons": [], + "rfc_number": 9999, + "final_approval": [{"approved": True}], + } + ] + ) + + url_obj = draft.documenturl_set.filter(tag_id="auth48").first() + self.assertIsNotNone(url_obj) + self.assertEqual(url_obj.url, "https://queue.example.com/final-review/rfc9999/") + + @override_settings(RFC_EDITOR_QUEUE_SITE_BASE_URL="https://queue.example.com") + def test_auth48_url_not_created_without_rfc_number(self): + """No auth48 URL created when rfc_number is None even if final_approval is set.""" + draft = WgDraftFactory() + + tasks.process_rpc_queue_task( + [ + { + "name": draft.name, + "assignment_set": [ + {"role": "first_editor", "state": "in_progress"} + ], + "blocking_reasons": [], + "rfc_number": None, + "final_approval": [{"approved": True}], + } + ] + ) + + self.assertFalse(draft.documenturl_set.filter(tag_id="auth48").exists()) + + @override_settings(RFC_EDITOR_QUEUE_SITE_BASE_URL="https://queue.example.com") + def test_auth48_url_deleted_when_final_approval_cleared(self): + """Existing auth48 URL is deleted whenever final_approval is empty, regardless of whether assignments changed.""" + draft = WgDraftFactory() + DocumentURL.objects.create( + doc=draft, + tag_id="auth48", + url="https://queue.example.com/final-review/rfc9999/", + ) + RpcAssignmentDocEvent.objects.create( + doc=draft, + rev=draft.rev, + by=self.system, + type="changed_rpc_assignments", + assignments="old_editor", + desc="RPC status changed to old_editor", + ) + + tasks.process_rpc_queue_task([_make_entry(draft.name, roles=["first_editor"])]) + + self.assertFalse(draft.documenturl_set.filter(tag_id="auth48").exists()) + + @override_settings(RFC_EDITOR_QUEUE_SITE_BASE_URL="https://queue.example.com") + def test_auth48_url_updated_when_rfc_number_changes(self): + """auth48 URL is updated whenever final_approval and rfc_number are set, regardless of whether assignments changed.""" + draft = WgDraftFactory() + DocumentURL.objects.create( + doc=draft, + tag_id="auth48", + url="https://queue.example.com/final-review/rfc8888/", + ) + RpcAssignmentDocEvent.objects.create( + doc=draft, + rev=draft.rev, + by=self.system, + type="changed_rpc_assignments", + assignments="old_editor", + desc="RPC status changed to old_editor", + ) + + tasks.process_rpc_queue_task( + [ + { + "name": draft.name, + "assignment_set": [ + {"role": "first_editor", "state": "in_progress"} + ], + "blocking_reasons": [], + "rfc_number": 9999, + "final_approval": [{"approved": True}], + } + ] + ) + + url_obj = draft.documenturl_set.filter(tag_id="auth48").first() + self.assertIsNotNone(url_obj) + self.assertEqual(url_obj.url, "https://queue.example.com/final-review/rfc9999/") + + @override_settings(RFC_EDITOR_QUEUE_SITE_BASE_URL="https://queue.example.com") + def test_auth48_url_created_when_assignments_unchanged(self): + """auth48 URL is created even when assignments have not changed.""" + draft = WgDraftFactory() + RpcAssignmentDocEvent.objects.create( + doc=draft, + rev=draft.rev, + by=self.system, + type="changed_rpc_assignments", + assignments="first_editor", + desc="RPC status changed to first_editor", + ) + + tasks.process_rpc_queue_task( + [ + { + "name": draft.name, + "assignment_set": [ + {"role": "first_editor", "state": "in_progress"} + ], + "blocking_reasons": [], + "rfc_number": 9999, + "final_approval": [{"approved": True}], + } + ] + ) + + url_obj = draft.documenturl_set.filter(tag_id="auth48").first() + self.assertIsNotNone(url_obj) + self.assertEqual(url_obj.url, "https://queue.example.com/final-review/rfc9999/") + + @override_settings(RFC_EDITOR_QUEUE_SITE_BASE_URL="https://queue.example.com") + def test_auth48_url_deleted_when_assignments_unchanged(self): + """Existing auth48 URL is deleted even when assignments have not changed.""" + draft = WgDraftFactory() + DocumentURL.objects.create( + doc=draft, + tag_id="auth48", + url="https://queue.example.com/final-review/rfc9999/", + ) + RpcAssignmentDocEvent.objects.create( + doc=draft, + rev=draft.rev, + by=self.system, + type="changed_rpc_assignments", + assignments="first_editor", + desc="RPC status changed to first_editor", + ) + + tasks.process_rpc_queue_task([_make_entry(draft.name, roles=["first_editor"])]) + + self.assertFalse(draft.documenturl_set.filter(tag_id="auth48").exists()) + + # --- Tag handling ------------------------------------------------------------ + + def test_removes_iana_and_ref_tags_from_queued_docs(self): + """iana and ref tags are removed from documents in the queue.""" + iana_tag = DocTagName.objects.get(slug="iana") + ref_tag = DocTagName.objects.get(slug="ref") + draft = WgDraftFactory() + draft.tags.add(iana_tag, ref_tag) + + tasks.process_rpc_queue_task([_make_entry(draft.name)]) + + draft = Document.objects.get(pk=draft.pk) + self.assertNotIn(iana_tag, draft.tags.all()) + self.assertNotIn(ref_tag, draft.tags.all()) + + # --- Cleanup of docs no longer in queue -------------------------------------- + + def test_unsets_rfceditor_state_for_docs_not_in_queue(self): + """Documents with draft-rfceditor state but absent from the queue have that state cleared.""" + draft = WgDraftFactory(states=[("draft-rfceditor", "in_progress")]) + + tasks.process_rpc_queue_task([]) + + draft = Document.objects.get(pk=draft.pk) + self.assertIsNone(draft.get_state("draft-rfceditor")) + + def test_removes_tags_from_docs_not_in_queue(self): + """iana and ref tags are removed from docs with rfceditor state not in the queue.""" + iana_tag = DocTagName.objects.get(slug="iana") + ref_tag = DocTagName.objects.get(slug="ref") + draft = WgDraftFactory(states=[("draft-rfceditor", "in_progress")]) + draft.tags.add(iana_tag, ref_tag) + + tasks.process_rpc_queue_task([]) + + draft = Document.objects.get(pk=draft.pk) + self.assertNotIn(iana_tag, draft.tags.all()) + self.assertNotIn(ref_tag, draft.tags.all()) + + def test_docs_in_queue_retain_rfceditor_state(self): + """Documents present in the queue keep their draft-rfceditor state.""" + draft = WgDraftFactory(states=[("draft-rfceditor", "in_progress")]) + + tasks.process_rpc_queue_task([_make_entry(draft.name, roles=["first_editor"])]) + + draft = Document.objects.get(pk=draft.pk) + self.assertIsNotNone(draft.get_state("draft-rfceditor")) diff --git a/ietf/sync/tests_utils.py b/ietf/sync/tests_utils.py new file mode 100644 index 0000000000..bb4a859e30 --- /dev/null +++ b/ietf/sync/tests_utils.py @@ -0,0 +1,84 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from pathlib import Path +from tempfile import TemporaryDirectory + +from django.test import override_settings +from ietf import settings +from ietf.doc.factories import RfcFactory +from ietf.doc.storage_utils import exists_in_storage, retrieve_str +from ietf.sync.utils import build_from_file_content, load_rfcs_into_blobdb, rsync_helper +from ietf.utils.test_utils import TestCase + + +class RsyncHelperTests(TestCase): + def test_rsync_helper(self): + with ( + TemporaryDirectory() as source_dir, + TemporaryDirectory() as dest_dir, + ): + with (Path(source_dir) / "canary.txt").open("w") as canary_source_file: + canary_source_file.write("chirp") + rsync_helper( + [ + "-a", + f"{source_dir}/", + f"{dest_dir}/", + ] + ) + with (Path(dest_dir) / "canary.txt").open("r") as canary_dest_file: + chirp = canary_dest_file.read() + self.assertEqual(chirp, "chirp") + + def test_build_from_file_content(self): + content = build_from_file_content([12345, 54321]) + self.assertEqual( + content, + """prerelease/ +rfc12345.txt +rfc12345.html +rfc12345.xml +rfc12345.pdf +rfc12345.ps +rfc12345.json +prerelease/rfc12345.notprepped.xml +rfc54321.txt +rfc54321.html +rfc54321.xml +rfc54321.pdf +rfc54321.ps +rfc54321.json +prerelease/rfc54321.notprepped.xml +""", + ) + + +class RfcBlobUploadTests(TestCase): + def test_load_rfcs_into_blobdb(self): + with TemporaryDirectory() as faux_rfc_path: + with override_settings(RFC_PATH=faux_rfc_path): + rfc_path = Path(faux_rfc_path) + (rfc_path / "prerelease").mkdir() + for num in [12345, 54321]: + RfcFactory(rfc_number=num) + for ext in settings.RFC_FILE_TYPES + ("json",): + with (rfc_path / f"rfc{num}.{ext}").open("w") as f: + f.write(ext) + with (rfc_path / "rfc{num}.bogon").open("w") as f: + f.write("bogon") + with (rfc_path / "prerelease" / f"rfc{num}.notprepped.xml").open( + "w" + ) as f: + f.write("notprepped") + load_rfcs_into_blobdb([12345, 54321]) + for num in [12345, 54321]: + for ext in settings.RFC_FILE_TYPES + ("json",): + self.assertEqual( + retrieve_str("rfc", f"{ext}/rfc{num}.{ext}"), + ext, + ) + self.assertFalse(exists_in_storage("rfc", f"bogon/rfc{num}.bogon")) + self.assertEqual( + retrieve_str("rfc", f"notprepped/rfc{num}.notprepped.xml"), + "notprepped", + ) diff --git a/ietf/sync/utils.py b/ietf/sync/utils.py new file mode 100644 index 0000000000..b3bdd8d206 --- /dev/null +++ b/ietf/sync/utils.py @@ -0,0 +1,77 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +import datetime +import subprocess + +from pathlib import Path + +from django.conf import settings +from ietf.utils import log +from ietf.doc.models import Document +from ietf.doc.storage_utils import AlreadyExistsError, store_bytes + + +def rsync_helper(subprocess_arg_array: list[str]): + subprocess.run(["/usr/bin/rsync"]+subprocess_arg_array) + +def build_from_file_content(rfc_numbers: list[int]) -> str: + types_to_sync = settings.RFC_FILE_TYPES + ("json",) + lines = [] + lines.append("prerelease/") + for num in rfc_numbers: + for ext in types_to_sync: + lines.append(f"rfc{num}.{ext}") + lines.append(f"prerelease/rfc{num}.notprepped.xml") + return "\n".join(lines)+"\n" + +def load_rfcs_into_blobdb(numbers: list[int]): + types_to_load = settings.RFC_FILE_TYPES + ("json",) + rfc_docs = Document.objects.filter(type="rfc", rfc_number__in=numbers).values_list("rfc_number", flat=True) + for num in numbers: + if num in rfc_docs: + for ext in types_to_load: + fs_path = Path(settings.RFC_PATH) / f"rfc{num}.{ext}" + if fs_path.is_file(): + with fs_path.open("rb") as f: + bytes = f.read() + mtime = fs_path.stat().st_mtime + try: + store_bytes( + kind="rfc", + name=f"{ext}/rfc{num}.{ext}", + content=bytes, + allow_overwrite=False, # Intentionally not allowing overwrite. + doc_name=f"rfc{num}", + doc_rev=None, + # Not setting content_type + mtime=datetime.datetime.fromtimestamp( + mtime, tz=datetime.UTC + ), + ) + except AlreadyExistsError as e: + log.log(str(e)) + + # store the not-prepped xml + name = f"rfc{num}.notprepped.xml" + source = Path(settings.RFC_PATH) / "prerelease" / name + if source.is_file(): + with open(source, "rb") as f: + bytes = f.read() + mtime = source.stat().st_mtime + try: + store_bytes( + kind="rfc", + name=f"notprepped/{name}", + content=bytes, + allow_overwrite=False, # Intentionally not allowing overwrite. + doc_name=f"rfc{num}", + doc_rev=None, + # Not setting content_type + mtime=datetime.datetime.fromtimestamp(mtime, tz=datetime.UTC), + ) + except AlreadyExistsError as e: + log.log(str(e)) + else: + log.log( + f"Skipping loading rfc{num} into blobdb as no matching Document exists" + ) diff --git a/ietf/templates/admin/meeting/Registration/change_list.html b/ietf/templates/admin/meeting/Registration/change_list.html new file mode 100644 index 0000000000..62784b2cb6 --- /dev/null +++ b/ietf/templates/admin/meeting/Registration/change_list.html @@ -0,0 +1,10 @@ +{% extends "admin/change_list.html" %} + +{% block search %} + {{ block.super }} {# This includes the original search form #} + {% if cl.search_fields %} {# Only show if search is enabled for the model #} +

    + Hint: Search by: {{ cl.search_fields|join:", "|capfirst }}. +

    + {% endif %} +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/admin/meeting/RegistrationTicket/change_list.html b/ietf/templates/admin/meeting/RegistrationTicket/change_list.html new file mode 100644 index 0000000000..62784b2cb6 --- /dev/null +++ b/ietf/templates/admin/meeting/RegistrationTicket/change_list.html @@ -0,0 +1,10 @@ +{% extends "admin/change_list.html" %} + +{% block search %} + {{ block.super }} {# This includes the original search form #} + {% if cl.search_fields %} {# Only show if search is enabled for the model #} +

    + Hint: Search by: {{ cl.search_fields|join:", "|capfirst }}. +

    + {% endif %} +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/base.html b/ietf/templates/base.html index aa44955527..b0df04f30a 100644 --- a/ietf/templates/base.html +++ b/ietf/templates/base.html @@ -15,6 +15,7 @@ {% block title %}No title{% endblock %} + @@ -66,13 +67,17 @@ {% endif %} - +
    • {% endif %}
  • - Session requests @@ -324,7 +344,7 @@
    • {% endif %}
  • - Upcoming meetings @@ -339,13 +359,13 @@
    • {% endif %}
  • - Past meetings
  • - Meeting proceedings @@ -371,57 +391,64 @@
    • {% endif %}
  • - IPR disclosures
  • - Liaison statements
    • + {% if user|has_role:"Secretariat,IAB,Liaison Manager,Liaison Coordinator" %}
  • - + List of other SDO groups + +
    • + {% endif %} +
  • + IESG agenda
  • - NomComs
  • - Downref registry
    • -
  • - +
  • + Statistics -
      +
    • - API Help
    • - Release notes
    • - System status diff --git a/ietf/templates/base/menu_user.html b/ietf/templates/base/menu_user.html index 9a0bf56838..fd921638a4 100644 --- a/ietf/templates/base/menu_user.html +++ b/ietf/templates/base/menu_user.html @@ -115,6 +115,37 @@
      • {% endif %} +
    • + + List subscriptions + + +
      • {% if user|has_role:"Reviewer" %}
    • IESG
      • + {% endif%}
    • diff --git a/ietf/templates/community/atom.xml b/ietf/templates/community/atom.xml index 32e3b00292..01dcdfeee7 100644 --- a/ietf/templates/community/atom.xml +++ b/ietf/templates/community/atom.xml @@ -3,7 +3,7 @@ {{ title }} {{ subtitle }} {{ id }} - {{ updated|date:"Y-m-d\TH:i:sO" }} + {{ updated.isoformat }} @@ -17,11 +17,11 @@ - {{ entry.id }} + urn:datatracker-ietf-org:event:{{ entry.id }} - {{ entry.time|date:"Y-m-d\TH:i:sO" }} + {{ entry.time.isoformat }} - {{ entry.time|date:"Y-m-d\TH:i:sO" }} + {{ entry.time.isoformat }} {{ entry.by }} diff --git a/ietf/templates/doc/ad_list.html b/ietf/templates/doc/ad_list.html index a73264c0f3..cac709021e 100644 --- a/ietf/templates/doc/ad_list.html +++ b/ietf/templates/doc/ad_list.html @@ -8,15 +8,19 @@ {% endblock %} {% block morecss %} table .border-bottom { border-bottom-color: var(--highcharts-neutral-color-80) !important; } - .highcharts-container .highcharts-axis-labels { font-size: .7rem; } + .highcharts-container .highcharts-axis-labels { + font-size: .7rem; + fill: var(--bs-body-color) + } .highcharts-container .highcharts-graph { stroke-width: 2.5; } .highcharts-container .highcharts-color-0 { - fill: var(--bs-primary); + fill: var(--bs-body-color); stroke: var(--bs-primary); } .highcharts-container .highcharts-data-label text { font-size: 1rem; font-weight: inherit; + fill: var(--bs-body-color) } {% endblock %} {% block title %}IESG Dashboard{% endblock %} @@ -29,6 +33,10 @@

      IESG Dashboard

      are only shown to logged-in Area Directors. {% endif %} +

      + Documents in IESG Processing + IESG view of Working Groups +

      {% for dt in metadata %}

      {{ dt.type.1 }} State Counts

      diff --git a/ietf/templates/doc/ballot/edit_position.html b/ietf/templates/doc/ballot/edit_position.html index 293c186112..b57e9a3652 100644 --- a/ietf/templates/doc/ballot/edit_position.html +++ b/ietf/templates/doc/ballot/edit_position.html @@ -20,24 +20,48 @@

      Ballot deferred by {{ ballot_deferred.by }} on {{ ballot_deferred.time|date:"Y-m-d" }}.

      {% endif %} +
      +
      + {% if form.errors or cc_select_form.errors or additional_cc_form.errors %} +
      + There were errors in the submitted form -- see below. Please correct these and resubmit. +
      + {% if form.errors %} +
      Position entry
      + {% bootstrap_form_errors form %} + {% endif %} + {% if cc_select_form.errors %} +
      CC selection
      + {% bootstrap_form_errors cc_select_form %} + {% endif %} + {% if additional_cc_form.errors %} +
      Additional Cc Addresses
      + {% bootstrap_form_errors additional_cc_form %} + {% endif %} + {% endif %}
      {% csrf_token %} {% for field in form %} {% if field.name == "discuss" %}
      {% endif %} {% bootstrap_field field %} {% if field.name == "discuss" and old_pos and old_pos.discuss_time %} -
      Last edited {{ old_pos.discuss_time }}
      +
      Last saved {{ old_pos.discuss_time }}
      {% elif field.name == "comment" and old_pos and old_pos.comment_time %} -
      Last edited {{ old_pos.comment_time }}
      +
      Last saved {{ old_pos.comment_time }}
      {% endif %} {% if field.name == "discuss" %}
      {% endif %} {% endfor %} + {% bootstrap_form cc_select_form %} + {% bootstrap_form additional_cc_form %}
      + - + {% if doc.type_id == "draft" or doc.type_id == "conflrev" %} {% if doc.stream.slug != "irtf" %} {% if ballot_deferred %} @@ -58,7 +82,24 @@

      Back

      - + + + {% endblock %} {% block js %} + + {% endblock %} \ No newline at end of file diff --git a/ietf/templates/doc/ballot/send_ballot_comment.html b/ietf/templates/doc/ballot/send_ballot_comment.html deleted file mode 100644 index 1c5f521859..0000000000 --- a/ietf/templates/doc/ballot/send_ballot_comment.html +++ /dev/null @@ -1,44 +0,0 @@ -{% extends "base.html" %} -{# Copyright The IETF Trust 2015, All Rights Reserved #} -{% load origin %} -{% load django_bootstrap5 %} -{% load ietf_filters %} -{% block title %}Send ballot position for {{ balloter }} on {{ doc }}{% endblock %} -{% block content %} - {% origin %} -

      - Send ballot position for {{ balloter }} -
      - {{ doc }} -

      - - {% csrf_token %} -
      - - -
      -
      - - -
      - {% bootstrap_form cc_select_form %} -
      - - -
      Separate email addresses with commas.
      -
      -
      - - -
      -
      -

      Body

      - 
      {{ body|maybewordwrap }}
      -
      - - - Back - - -{% endblock %} diff --git a/ietf/templates/doc/bofreq/bofreq_template.md b/ietf/templates/doc/bofreq/bofreq_template.md index 78949ba7bd..49c5e511a5 100644 --- a/ietf/templates/doc/bofreq/bofreq_template.md +++ b/ietf/templates/doc/bofreq/bofreq_template.md @@ -1,15 +1,15 @@ -# Name: Exact MPLS Edges (EXAMPLE) (There's an acronym for anything if you really want one ;-) +# Name: EXAct MPLs Edges (EXAMPLE) (There's an acronym for anything if you really want one ;-) ## Description Replace this with a few paragraphs describing the BOF request. Fill in the details below. Keep items in the order they appear here. ## Required Details -- Status: (not) WG Forming -- Responsible AD: name +- Status: "not WG Forming" or "WG forming" +- Responsible AD: name (or at least area(s) if you know) - BOF proponents: name , name (1-3 people - who are requesting and coordinating discussion for proposal) - Number of people expected to attend: 100 -- Length of session (1 or 2 hours): 2 hours +- Length of session (1 or usually 2 hours): 2 hours - Conflicts (whole Areas and/or WGs) - Chair Conflicts: TBD - Technology Overlap: TBD @@ -27,7 +27,7 @@ To allow evaluation of your proposal, please include the following items: - Items, Internet-Drafts, speakers, timing - Or a URL -## Links to the mailing list, draft charter if any, relevant Internet-Drafts, etc. +## Links to the mailing list, draft charter if any (for WG-forming BoF), relevant Internet-Drafts, etc. - Mailing List: https://www.ietf.org/mailman/listinfo/example - Draft charter: {{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.doc.views_doc.document_main' name='charter-ietf-EXAMPLE' %} - Relevant Internet-Drafts: diff --git a/ietf/templates/doc/bofreq/new_bofreq.html b/ietf/templates/doc/bofreq/new_bofreq.html index 9506d9c8e6..cda6f73b90 100644 --- a/ietf/templates/doc/bofreq/new_bofreq.html +++ b/ietf/templates/doc/bofreq/new_bofreq.html @@ -1,20 +1,29 @@ {% extends "base.html" %} -{# Copyright The IETF Trust 2021, All Rights Reserved #} +{# Copyright The IETF Trust 2021-2026, All Rights Reserved #} {% load origin django_bootstrap5 static textfilters %} {% block title %}Start a new BOF Request{% endblock %} {% block content %} {% origin %}

      Start a new BOF Request

      - The IAB will also attempt to provide BoF Shepherds as described in their document on the subject only on request from the IESG. If you feel that your BoF would benefit from an IAB BoF Shepherd, please discuss this with your Area Director. + BoF proponents are strongly encouraged to review the following sources before submitting requests:

      +

      - Choose a short descriptive title for your request. Take time to choose a good initial title - it will be used to make the filename for your request's content. The title can be changed later, but the filename will not change. + The IAB will also attempt to provide BoF Shepherds as described in their document on the subject only on request from the IESG. + If you feel that your BoF would benefit from an IAB BoF Shepherd, please discuss this with your Area Director. +

      +

      + Choose a short descriptive title for your request. Take time to choose a good initial title - it will be used to make the filename for your request's content. + The title can be changed later, but the filename will not change.

      For example, a request with a title of "A new important bit" will be saved as bofreq-{{ user.person.last_name|xslugify|slice:"64" }}-a-new-important-bit-00.md.

      -

      All the items in the template MUST be filed in.

      +

      All the items in the template MUST be filed in.

      diff --git a/ietf/templates/doc/document_draft.html b/ietf/templates/doc/document_draft.html index d5ea074422..eab1d779fb 100644 --- a/ietf/templates/doc/document_draft.html +++ b/ietf/templates/doc/document_draft.html @@ -63,7 +63,12 @@       - + + {% endif %} - {% if review_req.doc.authors %} + {% if review_req.doc.author_persons_or_names %} diff --git a/ietf/templates/doc/statement/change_statement_state.html b/ietf/templates/doc/statement/change_statement_state.html new file mode 100644 index 0000000000..aa5cb934e8 --- /dev/null +++ b/ietf/templates/doc/statement/change_statement_state.html @@ -0,0 +1,22 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% extends "base.html" %} +{% load origin %} +{% load django_bootstrap5 %} +{% block title %}Change state for {{ statement }}{% endblock %} +{% block content %} + {% origin %} +

      + Change state +
      + {{ statement }} +

      + + {% csrf_token %} + {% bootstrap_form form %} + + + Back + + +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/group/active_teams.html b/ietf/templates/group/active_teams.html index 502d971a20..771dfda290 100644 --- a/ietf/templates/group/active_teams.html +++ b/ietf/templates/group/active_teams.html @@ -16,21 +16,29 @@

      Active teams

      - - {% for group in teams %} - - - - - - {% endfor %} - + {% regroup teams by parent as grouped_teams %} + {% for group_entry in grouped_teams %} + + + + + {% for group in group_entry.list %} + + + + + + {% endfor %} + + {% endfor %}
      {% if doc.stream and can_edit_stream_info and doc.stream.slug != "legacy" and not snapshot %} + {% if doc|is_doc_ietf_adoptable or doc|can_issue_ietf_wg_lc or doc|can_submit_to_iesg %} + href="{% url 'ietf.doc.views_draft.offer_wg_action_helpers' name=doc.name %}" + {% else %} + href="{% url 'ietf.doc.views_draft.change_stream_state' name=doc.name state_type=stream_state_type_slug %}" + {% endif %} + > Edit {% endif %} @@ -661,6 +666,17 @@ Nits + {% if user|has_role:"Area Director" %} + {# IDNITS3 is an experimental service, so only show it to Area Directors #} + + + + Nits-v3 (Experimental) + + {% endif %} Revision differences {% include "doc/document_history_form.html" with doc=doc diff_revisions=diff_revisions action=rfcdiff_base_url snapshot=snapshot only %} {% endif %} + {% if ballot_doc_rev %} +

      + Your most recent ballot position was provided when the document was at version {{ ballot_doc_rev }}. +

      + {% endif %}

      Document history

      {% if doc.came_from_draft %}
        diff --git a/ietf/templates/doc/document_info.html b/ietf/templates/doc/document_info.html index 71050f9d41..1666c42ae5 100644 --- a/ietf/templates/doc/document_info.html +++ b/ietf/templates/doc/document_info.html @@ -87,7 +87,7 @@ {% endif %}
      Author{% if doc.pk %}{{ doc.authors|pluralize }}{% endif %}Author{% if doc.pk %}{{ doc.author_persons_or_names|pluralize }}{% endif %} {% if can_edit_authors %} {# Implementation that uses the current primary email for each author #} - {% if doc.pk %}{% for author in doc.authors %} - {% person_link author %}{% if not forloop.last %},{% endif %} + {% if doc.pk %}{% for author in doc.author_persons_or_names %} + {% if author.person %}{% person_link author.person titlepage_name=author.titlepage_name %}{% else %}{{ author.titlepage_name }}{% endif %}{% if not forloop.last %},{% endif %} {% endfor %}{% endif %} {% if document_html and not snapshot or document_html and doc.rev == latest_rev%}
      diff --git a/ietf/templates/doc/document_rfc.html b/ietf/templates/doc/document_rfc.html index 7612ef8910..d4b309a964 100644 --- a/ietf/templates/doc/document_rfc.html +++ b/ietf/templates/doc/document_rfc.html @@ -124,6 +124,15 @@ Referenced by + {% if doc.rfc_number >= settings.FIRST_V3_RFC %} + + + + Get editor source + + {% endif %} State -       		{% if can_manage %} + + Edit + + {% endif %} {% if doc.get_state %} {{ doc.get_state.name }} diff --git a/ietf/templates/doc/draft/ask_about_ietf_adoption.html b/ietf/templates/doc/draft/ask_about_ietf_adoption.html new file mode 100644 index 0000000000..d19e4572b7 --- /dev/null +++ b/ietf/templates/doc/draft/ask_about_ietf_adoption.html @@ -0,0 +1,22 @@ +{% extends "base.html" %} +{# Copyright The IETF Trust 2015-2025, All Rights Reserved #} +{% load django_bootstrap5 ietf_filters origin %} +{% block title %}Manage Adoption of {{ doc }}{% endblock %} +{% block content %} + {% origin %} +

      + Manage Adoption +
      + {{ doc }} +

      +
      + Do you wish to issue an IETF Working Group call for adoption to one of these working groups? +
      + + {% csrf_token %} + {% bootstrap_form form %} + + No, I wish to manage adoption directly, perhaps with non-IETF-stream groups + Back + +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/doc/draft/change_stream_state.html b/ietf/templates/doc/draft/change_stream_state.html index 0b13e02fdf..7f3132b3c8 100644 --- a/ietf/templates/doc/draft/change_stream_state.html +++ b/ietf/templates/doc/draft/change_stream_state.html @@ -1,7 +1,6 @@ {% extends "base.html" %} -{# Copyright The IETF Trust 2015, All Rights Reserved #} -{% load origin %} -{% load django_bootstrap5 %} +{# Copyright The IETF Trust 2015-2025, All Rights Reserved #} +{% load django_bootstrap5 ietf_filters origin %} {% block title %}Change {{ state_type.label }} for {{ doc }}{% endblock %} {% block content %} {% origin %} @@ -14,12 +13,10 @@

      Help on states

      - Move document to {{ next_states|pluralize:"to one of" }} the recommended next state{{ next_states|pluralize }}: + Move document to {{ next_states|pluralize:"one of" }} the recommended next state{{ next_states|pluralize }}:

      {% for state in next_states %} - {% if state.slug == 'sub-pub' %} - {{ state.name }} - {% else %} + {% if state.slug != 'sub-pub' and state.slug != "wg-lc" %} {% endif %} {% endfor %} @@ -28,7 +25,13 @@

      {% csrf_token %} {% bootstrap_form form %} - Back + Back {% endblock %} {% block js %} diff --git a/ietf/templates/doc/draft/id_expired_email.txt b/ietf/templates/doc/draft/id_expired_email.txt index afbf253ee2..161146a301 100644 --- a/ietf/templates/doc/draft/id_expired_email.txt +++ b/ietf/templates/doc/draft/id_expired_email.txt @@ -1,4 +1,4 @@ -{% autoescape off %}{{ doc.file_tag|safe }} was just expired. +{% autoescape off %}{{ doc.file_tag|safe }} just expired. This Internet-Draft is in the state "{{ state }}" in the Datatracker. diff --git a/ietf/templates/doc/draft/issue_working_group_call_for_adoption.html b/ietf/templates/doc/draft/issue_working_group_call_for_adoption.html new file mode 100644 index 0000000000..61094b053a --- /dev/null +++ b/ietf/templates/doc/draft/issue_working_group_call_for_adoption.html @@ -0,0 +1,61 @@ +{% extends "base.html" %} +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% load origin django_bootstrap5 static %} +={% block title %}Issue Working Group Call for Adoption of {{ doc }}{% endblock %} +{% block pagehead %} + +{% endblock %} +{% block content %} + {% origin %} +

      + Issue Working Group Call for Adoption +
      + {{ doc }} +

      + {% if form.errors %} +

      + Please correct the following: +

      + {% endif %} + {% bootstrap_form_errors form %} +
      + {% csrf_token %} + {% bootstrap_form form %} + + + Back +
      +{% endblock %} +{% block js %} + + +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/doc/draft/issue_working_group_last_call.html b/ietf/templates/doc/draft/issue_working_group_last_call.html new file mode 100644 index 0000000000..d6f35a0e82 --- /dev/null +++ b/ietf/templates/doc/draft/issue_working_group_last_call.html @@ -0,0 +1,61 @@ +{% extends "base.html" %} +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% load origin django_bootstrap5 static %} +={% block title %}Issue Working Group Last Call for {{ doc }}{% endblock %} +{% block pagehead %} + +{% endblock %} +{% block content %} + {% origin %} +

      + Issue Working Group Last Call +
      + {{ doc }} +

      + {% if form.errors %} +

      + Please correct the following: +

      + {% endif %} + {% bootstrap_form_errors form %} +
      + {% csrf_token %} + {% bootstrap_form form %} + + + Back +
      +{% endblock %} +{% block js %} + + +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/doc/draft/rfceditor_post_approved_draft_failed.html b/ietf/templates/doc/draft/rfceditor_post_approved_draft_failed.html deleted file mode 100644 index f976ead926..0000000000 --- a/ietf/templates/doc/draft/rfceditor_post_approved_draft_failed.html +++ /dev/null @@ -1,26 +0,0 @@ -{% extends "base.html" %} -{# Copyright The IETF Trust 2015, All Rights Reserved #} -{% load origin %} -{% block title %}Posting approved I-D to RFC Editor failed{% endblock %} -{% block content %} - {% origin %} -

      Posting approved I-D to RFC Editor failed

      -

      - Sorry, when trying to notify the RFC Editor through HTTP, we hit an - error. -

      -

      - We have not changed the Internet-Draft state or sent the announcement - yet so if this is an intermittent error, you can go back and try - again. -

      -

      - The error was: {{ error }} -

      - {% if response %} -

      - The response from the RFC Editor was: - {{ response|linebreaksbr }} -

      - {% endif %} -{% endblock %} \ No newline at end of file diff --git a/ietf/templates/doc/draft/wg_action_helpers.html b/ietf/templates/doc/draft/wg_action_helpers.html new file mode 100644 index 0000000000..d21f3c0926 --- /dev/null +++ b/ietf/templates/doc/draft/wg_action_helpers.html @@ -0,0 +1,25 @@ +{% extends "base.html" %} +{# Copyright The IETF Trust 2015-2025, All Rights Reserved #} +{% load django_bootstrap5 ietf_filters origin %} +{% block title %}Change IETF WG state for {{ doc }}{% endblock %} +{% block content %} + {% origin %} +

      + Change IETF WG state +
      + {{ doc }} +

      +
      + {% if doc|is_doc_ietf_adoptable %} + Issue WG Call for Adoption + {% endif %} + {% if doc|can_issue_ietf_wg_lc %} + Issue{% if doc|has_had_ietf_wg_lc %} Another{% endif %} Working Group Last Call + {% endif %} + {% if doc|can_submit_to_iesg %} + Submit to IESG for Publication + {% endif %} + Set any WG state directly + Back +
      +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/doc/drafts_for_iesg.html b/ietf/templates/doc/drafts_for_iesg.html new file mode 100644 index 0000000000..d7a79fb714 --- /dev/null +++ b/ietf/templates/doc/drafts_for_iesg.html @@ -0,0 +1,20 @@ +{% extends "base.html" %} +{# Copyright The IETF Trust 2015, All Rights Reserved #} +{% load origin static %} +{% load cache %} +{% load ietf_filters %} +{% load person_filters %} +{% block pagehead %} + +{% endblock %} +{% block title %}Documents for the IESG{% endblock %} +{% block content %} + {% cache 300 ietf_doc_drafts_for_iesg using="slowpages" %} + {% origin %} +

      Documents for the IESG

      + {% include "doc/search/search_results.html" with start_table=True end_table=True %} + {% endcache %} +{% endblock %} +{% block js %} + +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/doc/drafts_in_iesg_process.html b/ietf/templates/doc/drafts_in_iesg_process.html deleted file mode 100644 index f128e528a5..0000000000 --- a/ietf/templates/doc/drafts_in_iesg_process.html +++ /dev/null @@ -1,82 +0,0 @@ -{% extends "base.html" %} -{# Copyright The IETF Trust 2015, All Rights Reserved #} -{% load origin %} -{% load ietf_filters static %} -{% load textfilters person_filters %} -{% block pagehead %} - -{% endblock %} -{% block title %}{{ title }}{% endblock %} -{% block content %} - {% origin %} -

      {{ title }}

      - - - - - - - - - - - {% for state, docs in grouped_docs %} - - - - - - - {% for doc in docs %} - - - - - - - - {% endfor %} - - {% endfor %} -
      Area - {% if state.slug == "lc" %} - Expires at - {% else %} - Date - {% endif %} - DocumentIntended levelAD
      {{ state.name }}
      - {% if doc.area_acronym %} - {{ doc.area_acronym }} - {% endif %} - - {% if state.slug == "lc" %} - {% if doc.lc_expires %}{{ doc.lc_expires|date:"Y-m-d" }}{% endif %} - {% else %} - {{ doc.time|date:"Y-m-d" }} - {% endif %} - - {{ doc.name }} -
      - {{ doc.title }} - {% if doc.action_holders_enabled and doc.action_holders.exists %} -
      - Action holder{{ doc.documentactionholder_set.all|pluralize }}: - {% for action_holder in doc.documentactionholder_set.all %} - {% person_link action_holder.person title=action_holder.role_for_doc %}{% if action_holder|action_holder_badge %} {{ action_holder|action_holder_badge }}{% endif %}{% if not forloop.last %},{% endif %} - {% endfor %} - {% endif %} - {% if doc.note %} -
      - Note: {{ doc.note|urlize_ietf_docs|linkify|linebreaksbr }} - {% endif %} -       		- {% if doc.intended_std_level %} - {{ doc.intended_std_level.name }} - {% else %} - (None) - {% endif %} - {% person_link doc.ad %}
      -{% endblock %} -{% block js %} - -{% endblock %} \ No newline at end of file diff --git a/ietf/templates/doc/index_active_drafts.html b/ietf/templates/doc/index_active_drafts.html index 06ea2c4ff5..607385f56f 100644 --- a/ietf/templates/doc/index_active_drafts.html +++ b/ietf/templates/doc/index_active_drafts.html @@ -29,7 +29,7 @@

      Active Internet-Drafts

      {% for group in groups %}

      {{ group.name }} ({{ group.acronym }})

      - {% for d in group.active_drafts %} + {% for d in group.active_drafts %}{# n.b., d is a dict, not a Document #}
      {{ d.title }}. diff --git a/ietf/templates/doc/mail/wg_call_for_adoption_issued.txt b/ietf/templates/doc/mail/wg_call_for_adoption_issued.txt new file mode 100644 index 0000000000..15ace9495b --- /dev/null +++ b/ietf/templates/doc/mail/wg_call_for_adoption_issued.txt @@ -0,0 +1,31 @@ +{% load ietf_filters %}{% load mail_filters %}{% autoescape off %}{% filter wordwrap:78 %}This message starts a {{group.acronym}} WG Call for Adoption of: +{{ doc.name }}-{{ doc.rev }} + + +This Working Group Call for Adoption ends on {{ end_date }} + +Abstract: +{{ doc.abstract }} + +Please reply to this message and indicate whether or not you support adoption of this Internet-Draft by the {{group.acronym}} WG. Comments to explain your preference are greatly appreciated. Please reply to all recipients of this message and include this message in your response. + +Authors, and WG participants in general, are reminded of the Intellectual Property Rights (IPR) disclosure obligations described in BCP 79 [2]. Appropriate IPR disclosures required for full conformance with the provisions of BCP 78 [1] and BCP 79 [2] must be filed, if you are aware of any. Sanctions available for application to violators of IETF IPR Policy can be found at [3]. + +Thank you. +[1] https://datatracker.ietf.org/doc/bcp78/ +[2] https://datatracker.ietf.org/doc/bcp79/ +[3] https://datatracker.ietf.org/doc/rfc6701/ + +The IETF datatracker status page for this Internet-Draft is: +{{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.doc.views_doc.document_main' name=doc.name %} +{% if doc.submission.xml_version == "3" %} +There is also an HTML version available at: +{{ settings.IETF_ID_ARCHIVE_URL }}{{ doc.name }}-{{ doc.rev }}.html{% else %} +There is also an HTMLized version available at: +{{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.doc.views_doc.document_html' name=doc.name rev=doc.rev %}{% endif %} +{% if doc.rev != "00" %} +A diff from the previous version is available at: +{{settings.RFCDIFF_BASE_URL}}?url2={{ doc.name }}-{{ doc.rev }} +{% endif %} +{% endfilter %} +{% endautoescape %} diff --git a/ietf/templates/doc/mail/wg_last_call_issued.txt b/ietf/templates/doc/mail/wg_last_call_issued.txt new file mode 100644 index 0000000000..114f8bc5e2 --- /dev/null +++ b/ietf/templates/doc/mail/wg_last_call_issued.txt @@ -0,0 +1,34 @@ +{% load ietf_filters %}{% load mail_filters %}{% autoescape off %}{% filter wordwrap:78 %}This message starts a WG Last Call for: +{{ doc.name }}-{{ doc.rev }} + +This Working Group Last Call ends on {{ end_date }} + +Abstract: +{{ doc.abstract }} + +File can be retrieved from: +{{ url }} + +Please review and indicate your support or objection to proceed with the publication of this document by replying to this email keeping {{ wg_list }} in copy. Objections should be explained and suggestions to resolve them are highly appreciated. + +Authors, and WG participants in general, are reminded of the Intellectual Property Rights (IPR) disclosure obligations described in BCP 79 [1]. Appropriate IPR disclosures required for full conformance with the provisions of BCP 78 [1] and BCP 79 [2] must be filed, if you are aware of any. Sanctions available for application to violators of IETF IPR Policy can be found at [3]. + +Thank you. + +[1] https://datatracker.ietf.org/doc/bcp78/ +[2] https://datatracker.ietf.org/doc/bcp79/ +[3] https://datatracker.ietf.org/doc/rfc6701/ + +The IETF datatracker status page for this Internet-Draft is: +{{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.doc.views_doc.document_main' name=doc.name %} +{% if doc.submission.xml_version == "3" %} +There is also an HTML version available at: +{{ settings.IETF_ID_ARCHIVE_URL }}{{ doc.name }}-{{ doc.rev }}.html{% else %} +There is also an HTMLized version available at: +{{ settings.IDTRACKER_BASE_URL }}{% url 'ietf.doc.views_doc.document_html' name=doc.name rev=doc.rev %}{% endif %} +{% if doc.rev != "00" %} +A diff from the previous version is available at: +{{settings.RFCDIFF_BASE_URL}}?url2={{ doc.name }}-{{ doc.rev }} +{% endif %} +{% endfilter %} +{% endautoescape %} diff --git a/ietf/templates/doc/notprepped_wrapper.html b/ietf/templates/doc/notprepped_wrapper.html new file mode 100644 index 0000000000..078db7bc51 --- /dev/null +++ b/ietf/templates/doc/notprepped_wrapper.html @@ -0,0 +1,26 @@ +{% extends "base.html" %} +{# Copyright The IETF Trust 2026, All Rights Reserved #} +{% load origin %} +{% block title %}RFC {{ rfc.rfc_number }} — Not-prepped XML{% endblock %} +{% block content %} + {% origin %} +

      RFC {{ rfc.rfc_number }} — Not-prepped XML

      +

      + The not-prepped XML + is the RFC XML v3 source for an RFC at the moment in the publication process + just before the prep tool was used to expand default + values, generate section numbers, resolve cross-references, and embed + boilerplate. +

      + It is useful for authors who want to begin a new draft based on + the RFC's text, such as when creating a bis-draft, and for tools that process + author-facing RFC XML. +

      +

      + + + Download not-prepped XML for RFC {{ rfc.rfc_number }} + +

      +{% endblock %} diff --git a/ietf/templates/doc/opengraph.html b/ietf/templates/doc/opengraph.html index 4fe39b6209..1c8c5abe91 100644 --- a/ietf/templates/doc/opengraph.html +++ b/ietf/templates/doc/opengraph.html @@ -1,4 +1,4 @@ -{# Copyright The IETF Trust 2016-2020, All Rights Reserved #} +{# Copyright The IETF Trust 2016-2025, All Rights Reserved #} {% load origin %} {% load static %} {% load ietf_filters %} @@ -36,7 +36,7 @@ {% else %}{# TODO: We need a card image for individual I-Ds. #} {% endif %} -{% if doc.pk %}{% for author in doc.documentauthor_set.all %} +{% if doc.pk %}{% for author_name in doc.author_names %} {% endfor %}{% endif %} {% if published %}{% endif %} {% if expires %}{% endif %} \ No newline at end of file diff --git a/ietf/templates/doc/review/complete_review.html b/ietf/templates/doc/review/complete_review.html index 091f30532e..8aeb1486a4 100644 --- a/ietf/templates/doc/review/complete_review.html +++ b/ietf/templates/doc/review/complete_review.html @@ -93,7 +93,7 @@

      Back {% endif %}
      - {% if mail_archive_query_urls %} + {% if mail_archive_query_data %}
      + value="{{ mail_archive_query_data.query_value }}">
      diff --git a/ietf/templates/doc/review/request_info.html b/ietf/templates/doc/review/request_info.html index 9ad126d59e..51aea10a02 100644 --- a/ietf/templates/doc/review/request_info.html +++ b/ietf/templates/doc/review/request_info.html @@ -74,13 +74,13 @@

      		{% person_link review_req.requested_by %}
      Authors - {% for author in review_req.doc.authors %} - {% person_link author %}{% if not forloop.last %},{% endif %} + {% for person, tp_name in review_req.doc.author_persons_or_names %} + {% if person %}{% person_link person %}{% else %}{{ tp_name }}{% endif %}{% if not forloop.last %},{% endif %} {% endfor %}
      Chairs
      - {{ group.acronym }} - {{ group.name }} - {% for chair in group.chairs %} - {% person_link chair.person %}{% if not forloop.last %},{% endif %} - {% endfor %} -
      + {% if group_entry.grouper %}{{ group_entry.grouper.name }}{% else %}Other{% endif %} +
      + {{ group.acronym }} + {{ group.name }} + {% for chair in group.chairs %} + {% person_link chair.person %}{% if not forloop.last %},{% endif %} + {% endfor %} +
      {% endblock %} {% block js %} diff --git a/ietf/templates/group/change_reviewer_settings.html b/ietf/templates/group/change_reviewer_settings.html index 9ecec5633c..75451fdd75 100644 --- a/ietf/templates/group/change_reviewer_settings.html +++ b/ietf/templates/group/change_reviewer_settings.html @@ -89,7 +89,7 @@

      Unavailable periods

      + value="add_period">Save new period

      History of settings

      diff --git a/ietf/templates/group/group_about.html b/ietf/templates/group/group_about.html index cbc2e11536..0a8b9194f2 100644 --- a/ietf/templates/group/group_about.html +++ b/ietf/templates/group/group_about.html @@ -51,6 +51,13 @@ {{ group.parent.name }} ({{ group.parent.acronym }}) + {% elif group.parent and group.type_id == "team" %} + Parent + + + {{ group.parent.name }} + ({{ group.parent.acronym }}) + {% else %} @@ -444,4 +451,4 @@

      group_stats("{% url 'ietf.group.views.group_stats_data' %}", ".chart"); }); -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/ietf/templates/group/manage_review_requests.html b/ietf/templates/group/manage_review_requests.html index 99b23c138a..d240ef24fa 100644 --- a/ietf/templates/group/manage_review_requests.html +++ b/ietf/templates/group/manage_review_requests.html @@ -66,10 +66,10 @@

      Auto-suggested
      {% endif %} - {% if r.doc.authors %} + {% if r.doc.author_persons_or_names %} Authors: - {% for person in r.doc.authors %} - {% person_link person %}{% if not forloop.last %},{% endif %} + {% for person, tp_name in r.doc.author_persons_or_names %} + {% if person %}{% person_link person %}{% else %}{{ tp_name }}{% endif %}{% if not forloop.last %},{% endif %} {% endfor %}
      {% endif %} diff --git a/ietf/templates/group/meetings-row.html b/ietf/templates/group/meetings-row.html index 25605ec0f1..8927eb61a2 100644 --- a/ietf/templates/group/meetings-row.html +++ b/ietf/templates/group/meetings-row.html @@ -1,3 +1,4 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} {% load origin tz %} {% origin %} {% for s in sessions %} @@ -25,7 +26,7 @@ {% if show_request and s.meeting.type_id == 'ietf' %} {% if can_edit %} + href="{% url 'ietf.meeting.views_session_request.view_request' num=s.meeting.number acronym=s.group.acronym %}"> Edit Session Request {% endif %} diff --git a/ietf/templates/group/meetings.html b/ietf/templates/group/meetings.html index deaea1e675..30f478da13 100644 --- a/ietf/templates/group/meetings.html +++ b/ietf/templates/group/meetings.html @@ -1,3 +1,4 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} {% extends "group/group_base.html" %} {% load origin static %} {% block title %} @@ -9,7 +10,7 @@ Session requests {% if can_edit or can_always_edit %} - Request a session + Request a session Request an interim meeting @@ -40,12 +41,12 @@

      Meetings in progress

      {% if future %}

      Future Meetings - - - + {% for cal_action in cal_actions %} + + {{ cal_action.label }} + + {% endfor %}

      diff --git a/ietf/templates/group/milestones.html b/ietf/templates/group/milestones.html index df03d370fd..51579ce80a 100644 --- a/ietf/templates/group/milestones.html +++ b/ietf/templates/group/milestones.html @@ -20,7 +20,7 @@

      - +{% regroup statements by status as grouped_statements %} +{% for statement_group in grouped_statements %} - {% for statement in statements %} + + + + + + {% for statement in statement_group.list %} - {% endfor %} + {% endfor %} +{% endfor %}
      + {% if group.uses_milestone_dates %} Date {% else %} @@ -39,7 +39,7 @@

      {{ milestone.resolved|title }} {% else %} {% if group.uses_milestone_dates %} - {{ milestone.due|date:"M Y" }} + {{ milestone.due|date:"M Y" }} {% else %} {% if forloop.first %}Last{% endif %} {% if forloop.last %}Next{% endif %} diff --git a/ietf/templates/group/review_requests_history.html b/ietf/templates/group/review_requests_history.html new file mode 100644 index 0000000000..1b1fb4d263 --- /dev/null +++ b/ietf/templates/group/review_requests_history.html @@ -0,0 +1,90 @@ +{% extends "group/group_base.html" %} +{# Copyright The IETF Trust 2015, All Rights Reserved #} +{% load origin %} +{% load tz %} +{% load ietf_filters person_filters textfilters %} +{% load static %} +{% block pagehead %} + +{% endblock %} +{% block group_content %} + {% origin %} + {% if reviewer_email %} +

      Review requests history of {{ reviewer_email }}

      + {% else %} +

      Review requests history

      + {% endif %} +
      +
      + + + +
      +
      + Past: +
      + {% for key, label in since_choices %} + + {% endfor %} +
      +
      +
      + + + + + + + + + + + + + + {% if history %} + + {% for h in history %} + + + + + + + + + + {% endfor %} + + {% endif %} +
      Date (UTC)ByDocumentStateReviewerResultDescription
      {{ h.history_date|utc|date:"Y-m-d H:i:s" }}{% person_link h.history_user.person %}{% if h.reviewed_rev %} + + {{ h.review_request.doc.name }}-{{ h.reviewed_rev }} + + {% else %} + {{ h.review_request.doc.name }} + {% endif %} + + {{ h.state }} + + {% person_link h.reviewer.person %} + + (set as filter) + + + {% if h.review %} + {{ h.result }} + {% else %} + {{ h.result }} + {% endif %} + {{ h.history_change_reason }}
      +{% endblock %} +{% block js %} + +{% endblock %} diff --git a/ietf/templates/group/statements.html b/ietf/templates/group/statements.html index 035c3bc967..6bbe3cb394 100644 --- a/ietf/templates/group/statements.html +++ b/ietf/templates/group/statements.html @@ -12,8 +12,8 @@

      {{group.acronym|upper}} Statements

      {% if request.user|has_role:"Secretariat" %}
      + class="btn btn-primary" + href="{% url 'ietf.doc.views_statement.new_statement' %}"> Start New Statement
      @@ -25,16 +25,25 @@

      {{group.acronym|upper}} Statements

      		Statement
      + {{ statement_group.grouper|title }} {{"Statement"|plural:statement_group.list }} ({{ statement_group.list|length }} {{"hit"|plural:statement_group.list }}) +
      {{ statement.published|date:"Y-m-d" }} {{statement.title}} - {% if statement.status == "replaced" %}Replaced{% endif %}
      {% endblock %} {% block js %} diff --git a/ietf/templates/iesg/working_groups.html b/ietf/templates/iesg/working_groups.html new file mode 100644 index 0000000000..b799636857 --- /dev/null +++ b/ietf/templates/iesg/working_groups.html @@ -0,0 +1,159 @@ +{% extends "base.html" %} +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% load origin static %} +{% block pagehead %} + +{% endblock %} +{% block title %}IESG view of working groups{% endblock %} +{% block content %} + {% origin %} +

      IESG view of working groups

      +

      Area Size and Load

      + + + + + + + + {# (divider) #} + + + + + + {% for area in area_summary %} + + + + + + + + + {% endfor %} + + + + + + + + + + + +
      Area NameWGsI-DsPages% I-Ds% Pages
      {{area.area}}{{area.groups_in_area}}{{area.doc_count}}{{area.page_count}}{{area.doc_percent|floatformat:1}}{{area.page_percent|floatformat:1}}
      Totals{{totals.group_count}}{{totals.doc_count}}{{totals.page_count}}
      + +

      Area Director Load: Documents not yet directly assigned to AD

      +
      Typically these are pre-pubreq documents
      + + + + + + + + + {# (divider) #} + + + + + + {% for ad in noad_summary %} + + + + + + + + + + {% endfor %} + + + + + + + + + + + + +
      ADArea NameWGs for ADI-DsPages% I-Ds% Pages
      {{ad.ad}}{{ad.area}}{{ad.ad_group_count}}{{ad.doc_count}}{{ad.page_count}}{{ad.doc_percent|floatformat:1}}{{ad.page_percent|floatformat:1}}
      Totals{{noad_totals.ad_group_count}}{{noad_totals.doc_count}}{{noad_totals.page_count}}
      + +

      Area Director Load: Documents directly assigned to AD

      + + + + + + + + + {# (divider) #} + + + + + + {% for ad in ad_summary %} + + + + + + + + + + {% endfor %} + + + + + + + + + + + + +
      ADArea NameWGs for ADI-DsPages% I-Ds% Pages
      {{ad.ad}}{{ad.area}}{{ad.ad_group_count}}{{ad.doc_count}}{{ad.page_count}}{{ad.doc_percent|floatformat:1}}{{ad.page_percent|floatformat:1}}
      Totals{{ad_totals.ad_group_count}}{{ad_totals.doc_count}}{{ad_totals.page_count}}
      + +

      Working Group Summary

      + + + + + + + + + + + + + + {% for wg in wg_summary %} + + + + + + + + + + {% endfor %} + +
      WGAreaADI-DsPagesRFCsRFCs in last 2 years
      {{wg.wg}}{{wg.area}}{{wg.ad}}{{wg.doc_count}}{{wg.page_count}}{{wg.rfc_count}}{{wg.recent_rfc_count}}
      +{% endblock %} +{% block js %} + +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/ipr/deleted.html b/ietf/templates/ipr/deleted.html new file mode 100644 index 0000000000..24f696ebca --- /dev/null +++ b/ietf/templates/ipr/deleted.html @@ -0,0 +1,16 @@ +{% extends "base.html" %} +{# Copyright The IETF Trust 2015-2023, All Rights Reserved #} +{% load ietf_filters origin %} +{% block title %}Removed IPR Disclosure{% endblock %} +{% block content %} + {% origin %} +

      Removed IPR disclosure

      +

      + {{ removed.reason }} +

      + {% if user|has_role:"Secretariat" and ipr.exists %} +

      + This disclosure has not yet been deleted and parts of its content is available through, e.g, the history view and the /api/v1 views. +

      + {% endif %} + {% endblock %} \ No newline at end of file diff --git a/ietf/templates/liaisons/detail.html b/ietf/templates/liaisons/detail.html index 4bf5b4d11b..d46c8d1c98 100644 --- a/ietf/templates/liaisons/detail.html +++ b/ietf/templates/liaisons/detail.html @@ -32,7 +32,7 @@

      {% if liaison.from_contact %} From Contact - {% person_link liaison.from_contact.person %} + {{ liaison.from_contact }} {% endif %} diff --git a/ietf/templates/liaisons/liaison_mail.txt b/ietf/templates/liaisons/liaison_mail.txt index c92c68ff76..18dfe610fd 100644 --- a/ietf/templates/liaisons/liaison_mail.txt +++ b/ietf/templates/liaisons/liaison_mail.txt @@ -1,13 +1,20 @@ -{% load ietf_filters %}{% autoescape off %}Title: {{ liaison.title|clean_whitespace }} +{% load ietf_filters group_filters %}{% autoescape off %}Title: {{ liaison.title|clean_whitespace }} Submission Date: {{ liaison.submitted|date:"Y-m-d" }} URL of the IETF Web page: {{ liaison.get_absolute_url }} + +To: {% for g in liaison.to_groups.all %}{{g|name_with_conditional_acronym}}{% if not forloop.last %}, {% endif %}{% endfor %} +From: {% for g in liaison.from_groups.all %}{{g|name_with_conditional_acronym}}{% if not forloop.last %}, {% endif %}{% endfor %} +Purpose: {{ liaison.purpose.name }} {% if liaison.deadline %}Please reply by {{ liaison.deadline }}{% endif %} -From: {% if liaison.from_contact %}{{ liaison.from_contact.formatted_email }}{% endif %} + +Email Addresses +--------------- +From: {% if liaison.from_contact %}{{ liaison.from_contact }}{% endif %} To: {{ liaison.to_contacts }} Cc: {{ liaison.cc_contacts }} Response Contacts: {{ liaison.response_contacts }} Technical Contacts: {{ liaison.technical_contacts }} -Purpose: {{ liaison.purpose.name }} + {% for related in liaison.source_of_set.all %} Referenced liaison: {% if related.target.title %}{{ related.target.title }}{% else %}Liaison #{{ related.target.pk }}{% endif %} ({{ related.target.get_absolute_url }}) {% endfor %} diff --git a/ietf/templates/liaisons/list_other_sdo.html b/ietf/templates/liaisons/list_other_sdo.html new file mode 100644 index 0000000000..e6a567ae50 --- /dev/null +++ b/ietf/templates/liaisons/list_other_sdo.html @@ -0,0 +1,41 @@ +{% extends "base.html" %} +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% load origin static person_filters %} +{% block pagehead %} + +{% endblock %} +{% block title %} + Other SDO groups +{% endblock %} +{% block content %} + {% origin %} + {% regroup sdos by state.name as sdos_by_state %} + {% for sdos_in_state in sdos_by_state %} +

      {{sdos_in_state.grouper|capfirst}} SDO groups

      + + + + + + + + + + {% for group in sdos_in_state.list%} + + + + + + {% endfor %} + +
      AcronymNameLiaison Managers
      {{ group.acronym }}{{ group.name }} + {% for person in group.liaison_managers %} + {% person_link person %}{% if not forloop.last %}, {% endif %} + {% endfor %} +
      + {% endfor %} +{% endblock %} +{% block js %} + +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/meeting/activity_report.html b/ietf/templates/meeting/activity_report.html index 0de8b88c65..dbfc717dab 100644 --- a/ietf/templates/meeting/activity_report.html +++ b/ietf/templates/meeting/activity_report.html @@ -40,14 +40,14 @@

      {{ rfcs.count }} RFCs published this period

      Title - {% if rfcs %} + {% if rfcs|length > 0 %} {% for rfc in rfcs %} {{ rfc.doc.name|prettystdname }} - {{ rfc.doc.intended_std_level.name }} + {{ rfc.doc.std_level.name }} {{ rfc.doc.group.acronym }} diff --git a/ietf/templates/meeting/agenda.ics b/ietf/templates/meeting/agenda.ics deleted file mode 100644 index eb83dd479a..0000000000 --- a/ietf/templates/meeting/agenda.ics +++ /dev/null @@ -1,32 +0,0 @@ -{% load humanize tz %}{% autoescape off %}{% timezone schedule.meeting.tz %}{% with tzname=schedule.meeting.time_zone %}{% load ietf_filters textfilters %}{% load cache %}{% cache 1800 ietf_meeting_agenda_ics schedule.meeting.number request.path request.GET %}BEGIN:VCALENDAR -VERSION:2.0 -METHOD:PUBLISH -PRODID:-//IETF//datatracker.ietf.org ical agenda//EN -{% if tzname != "utc" and tzname != "gmt" %}{% firstof schedule.meeting.vtimezone "" %}{% endif %}{% for item in assignments %}BEGIN:VEVENT -UID:ietf-{{schedule.meeting.number}}-{{item.timeslot.pk}}-{{item.session.group.acronym}} -SUMMARY:{% if item.session.name %}{{item.session.name|ics_esc}}{% else %}{{item.session.group_at_the_time.acronym|lower}} - {{item.session.group_at_the_time.name}}{%endif%}{% if item.session.agenda_note %} ({{item.session.agenda_note}}){% endif %} -{% if item.timeslot.show_location %}LOCATION:{{item.timeslot.get_location}} -{% endif %}STATUS:{{item.session.ical_status}} -CLASS:PUBLIC -DTSTART{% ics_date_time item.timeslot.local_start_time tzname %} -DTEND{% ics_date_time item.timeslot.local_end_time tzname %} -DTSTAMP{% ics_date_time item.timeslot.modified|utc 'utc' %}{% if item.session.agenda %} -URL:{{item.session.agenda.get_versionless_href}}{% endif %} -DESCRIPTION:{{item.timeslot.name|ics_esc}}\n{% if item.session.agenda_note %} - Note: {{item.session.agenda_note|ics_esc}}\n{% endif %}{% if item.session.onsite_tool_url %} - \n - Onsite tool: {{ item.session.onsite_tool_url }}\n{% endif %}{% if item.session.video_stream_url %} - \n - Meetecho: {{ item.session.video_stream_url }}\n{% endif %}{% if item.timeslot.location.webex_url %} - \n - Webex: {{ item.timeslot.location.webex_url }}\n{% endif %}{% if item.session.remote_instructions %} - \n - Remote instructions: {{ item.session.remote_instructions }}\n{% endif %}{% if item.session.agenda %}{% with agenda=item.session.agenda %} - \n - {{agenda.type}} {{agenda.get_versionless_href}}\n{% endwith %}{% endif %} - \n - Session materials: {% absurl 'ietf.meeting.views.session_details' num=schedule.meeting.number acronym=item.session.group.acronym %}\n{% if schedule.meeting.get_number is not None %} - \n{# link agenda for ietf meetings #} - See in schedule: {% absurl 'agenda' num=schedule.meeting.number %}#row-{{ item.slug }}\n{% endif %} -END:VEVENT -{% endfor %}END:VCALENDAR{% endcache %}{% endwith %}{% endtimezone %}{% endautoescape %} diff --git a/ietf/templates/meeting/approve_proposed_slides.html b/ietf/templates/meeting/approve_proposed_slides.html index 37fb523394..204473f455 100644 --- a/ietf/templates/meeting/approve_proposed_slides.html +++ b/ietf/templates/meeting/approve_proposed_slides.html @@ -34,6 +34,6 @@

      + value="disapprove">Decline and Delete -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/ietf/templates/meeting/bluesheet.txt b/ietf/templates/meeting/bluesheet.txt index dd3bf36ac7..5b3960f3aa 100644 --- a/ietf/templates/meeting/bluesheet.txt +++ b/ietf/templates/meeting/bluesheet.txt @@ -1,7 +1,8 @@ -Bluesheet for {{session}} +{% autoescape off %}Bluesheet for {{session}} ======================================================================== {{ data|length }} attendees. {% for item in data %} {{ item.name }} {{ item.affiliation }}{% endfor %} +{% endautoescape %} diff --git a/ietf/templates/meeting/edit_meeting_schedule.html b/ietf/templates/meeting/edit_meeting_schedule.html index 462f97715d..a975c2e61a 100644 --- a/ietf/templates/meeting/edit_meeting_schedule.html +++ b/ietf/templates/meeting/edit_meeting_schedule.html @@ -223,6 +223,11 @@

      {{ p.acronym }} {% endfor %} + +
      {% if session_purposes|length > 1 %} diff --git a/ietf/templates/meeting/important_dates.ics b/ietf/templates/meeting/important_dates.ics deleted file mode 100644 index 35079e01eb..0000000000 --- a/ietf/templates/meeting/important_dates.ics +++ /dev/null @@ -1,5 +0,0 @@ -{% load humanize %}{% autoescape off %}{% load ietf_filters %}BEGIN:VCALENDAR -VERSION:2.0 -METHOD:PUBLISH -PRODID:-//IETF//datatracker.ietf.org ical importantdates//EN -{% for meeting in meetings %}{% include "meeting/important_dates_for_meeting.ics" %}{% endfor %}END:VCALENDAR{% endautoescape %} diff --git a/ietf/templates/meeting/important_dates_for_meeting.ics b/ietf/templates/meeting/important_dates_for_meeting.ics deleted file mode 100644 index df5fe46818..0000000000 --- a/ietf/templates/meeting/important_dates_for_meeting.ics +++ /dev/null @@ -1,23 +0,0 @@ -{% load tz ietf_filters %}{% for d in meeting.important_dates %}BEGIN:VEVENT -UID:ietf-{{ meeting.number }}-{{ d.name_id }}-{{ d.date.isoformat }} -SUMMARY:IETF {{ meeting.number }}: {{ d.name.name }} -CLASS:PUBLIC -DTSTART{% if not d.midnight_cutoff %};VALUE=DATE{% endif %}:{{ d.date|date:"Ymd" }}{% if d.midnight_cutoff %}235900Z{% endif %} -DTSTAMP{% ics_date_time meeting.cached_updated|utc 'utc' %} -TRANSP:TRANSPARENT -DESCRIPTION:{{ d.name.desc }}{% if first and d.name.slug == 'openreg' or first and d.name.slug == 'earlybird' %}\n - Register here: https://www.ietf.org/how/meetings/register/{% endif %}{% if d.name.slug == 'opensched' %}\n - To request a Working Group session, use the IETF Meeting Session Request Tool:\n - {{ request.scheme }}://{{ request.get_host}}{% url 'ietf.secr.sreq.views.main' %}\n - If you are working on a BOF request, it is highly recommended to tell the IESG\n - now by sending an email to iesg@ietf.org to get advance help with the request.{% endif %}{% if d.name.slug == 'cutoffwgreq' %}\n - To request a Working Group session, use the IETF Meeting Session Request Tool:\n - {{ request.scheme }}://{{ request.get_host }}{% url 'ietf.secr.sreq.views.main' %}{% endif %}{% if d.name.slug == 'cutoffbofreq' %}\n - To request a BOF, please see instructions on Requesting a BOF:\n - https://www.ietf.org/how/bofs/bof-procedures/{% endif %}{% if d.name.slug == 'idcutoff' %}\n - Upload using the I-D Submission Tool:\n - {{ request.scheme }}://{{ request.get_host }}{% url 'ietf.submit.views.upload_submission' %}{% endif %}{% if d.name.slug == 'draftwgagenda' or d.name.slug == 'revwgagenda' or d.name.slug == 'procsub' or d.name.slug == 'revslug' %}\n - Upload using the Meeting Materials Management Tool:\n - {{ request.scheme }}://{{ request.get_host }}{% url 'ietf.meeting.views.materials' num=meeting.number %}{% endif %} -END:VEVENT -{% endfor %} diff --git a/ietf/templates/meeting/previously_approved_slides.html b/ietf/templates/meeting/previously_approved_slides.html index 25a4c97863..95975cb63f 100644 --- a/ietf/templates/meeting/previously_approved_slides.html +++ b/ietf/templates/meeting/previously_approved_slides.html @@ -11,7 +11,7 @@

      {% if submission.status.slug == 'approved' %} approved {% else %} - rejected + declined {% endif %}

      @@ -19,7 +19,7 @@

      {% if submission.status.slug == 'approved' %} approved. {% else %} - rejected. + declined. {% endif %} No further action is needed.

      @@ -33,4 +33,4 @@

      return to this meeting session .

      -{% endblock %} \ No newline at end of file +{% endblock %} diff --git a/ietf/templates/meeting/proceedings_attendees.html b/ietf/templates/meeting/proceedings_attendees.html index 6c51a4546e..0c59d4ab15 100644 --- a/ietf/templates/meeting/proceedings_attendees.html +++ b/ietf/templates/meeting/proceedings_attendees.html @@ -3,6 +3,7 @@ {% load origin markup_tags static %} {% block pagehead %} + {% if chart_data %}{% endif %} {% endblock %} {% block title %}IETF {{ meeting.number }} proceedings{% endblock %} {% block content %} @@ -14,8 +15,52 @@

      Attendee list of IETF {{ meeting.number }} meeting

      - + + {% if chart_data %} +
      +
      +
      Onsite: {{ stats.onsite }}
      +
      Remote: {{ stats.remote }}
      +
      Total: {{ stats.total }}
      +
      + +
      + + + + {{ chart_data|json_script:"attendees-chart-data" }} + {% endif %}{# chart_data #} + {% if template %} + {{template|safe}} {% else %} @@ -29,13 +74,13 @@

      Attendee list of IETF {{ meeting.number }} meeting

      - {% for reg in meeting_registrations %} + {% for reg in registrations %} - + {% endfor %} @@ -44,4 +89,8 @@

      Attendee list of IETF {{ meeting.number }} meeting

      {% endblock %} {% block js %} -{% endblock %} \ No newline at end of file + {% if chart_data %} + + + {% endif %} +{% endblock %} diff --git a/ietf/templates/meeting/requests.html b/ietf/templates/meeting/requests.html index 3008ceb662..0abee95887 100644 --- a/ietf/templates/meeting/requests.html +++ b/ietf/templates/meeting/requests.html @@ -1,5 +1,5 @@ {% extends "base.html" %} -{# Copyright The IETF Trust 2015, All Rights Reserved #} +{# Copyright The IETF Trust 2015-2025, All Rights Reserved #} {% load origin %} {% load ietf_filters static person_filters textfilters %} {% block pagehead %} @@ -151,7 +151,7 @@

      {% endifchanged %}

      {% for day in time_slices %} {% endfor %} {% endif %} diff --git a/ietf/templates/meeting/upcoming.ics b/ietf/templates/meeting/upcoming.ics deleted file mode 100644 index 5eca7ec81d..0000000000 --- a/ietf/templates/meeting/upcoming.ics +++ /dev/null @@ -1,32 +0,0 @@ -{% load humanize tz %}{% autoescape off %}{% load ietf_filters textfilters %}BEGIN:VCALENDAR -VERSION:2.0 -METHOD:PUBLISH -PRODID:-//IETF//datatracker.ietf.org ical upcoming//EN -{{vtimezones}}{% for item in assignments %}BEGIN:VEVENT -UID:ietf-{{item.session.meeting.number}}-{{item.timeslot.pk}} -SUMMARY:{% if item.session.name %}{{item.session.group.acronym|lower}} - {{item.session.name|ics_esc}}{% else %}{{item.session.group.acronym|lower}} - {{item.session.group.name}}{%endif%} -{% if item.schedule.meeting.city %}LOCATION:{{item.schedule.meeting.city}},{{item.schedule.meeting.country}} -{% endif %}STATUS:{{item.session.ical_status}} -CLASS:PUBLIC -DTSTART{% ics_date_time item.timeslot.local_start_time item.schedule.meeting.time_zone %} -DTEND{% ics_date_time item.timeslot.local_end_time item.schedule.meeting.time_zone %} -DTSTAMP{% ics_date_time item.timeslot.modified|utc 'utc' %}{% if item.session.agenda %} -URL:{{item.session.agenda.get_href}}{% endif %} -DESCRIPTION:{% if item.timeslot.name %}{{item.timeslot.name|ics_esc}}\n{% endif %}{% if item.session.agenda_note %} - Note: {{item.session.agenda_note|ics_esc}}\n{% endif %}{% for material in item.session.materials.all %} - \n{{material.type}}{% if material.type.name != "Agenda" %} - ({{material.title|ics_esc}}){% endif %}: - {{material.get_href}}\n{% endfor %}{% if item.session.remote_instructions %} - Remote instructions: {{ item.session.remote_instructions }}\n{% endif %} -END:VEVENT -{% endfor %}{% for meeting in ietfs %}BEGIN:VEVENT -UID:ietf-{{ meeting.number }} -SUMMARY:IETF {{ meeting.number }}{% if meeting.city %} -LOCATION:{{ meeting.city }},{{ meeting.country }}{% endif %} -CLASS:PUBLIC -DTSTART;VALUE=DATE{% if meeting.time_zone %};TZID={{ meeting.time_zone|ics_esc }}{% endif %}:{{ meeting.date|date:"Ymd" }} -DTEND;VALUE=DATE{% if meeting.time_zone %};TZID={{ meeting.time_zone|ics_esc }}{% endif %}:{{ meeting.end_date|next_day|date:"Ymd" }} -DTSTAMP{% ics_date_time meeting.cached_updated|utc 'utc' %} -URL:{{ request.scheme }}://{{ request.get_host }}{% url 'agenda' num=meeting.number %} -END:VEVENT -{% endfor %}END:VCALENDAR{% endautoescape %} diff --git a/ietf/templates/minimal.html b/ietf/templates/minimal.html index 87f661f501..15c432505e 100644 --- a/ietf/templates/minimal.html +++ b/ietf/templates/minimal.html @@ -9,8 +9,8 @@ {{ title }} - - + + {# load this in the head, to prevent flickering #} diff --git a/ietf/templates/person/merge.html b/ietf/templates/person/merge.html index 36499ecdbc..5c3e6b0938 100644 --- a/ietf/templates/person/merge.html +++ b/ietf/templates/person/merge.html @@ -1,5 +1,5 @@ +{# Copyright The IETF Trust 2018-2025, All Rights Reserved #} {% extends "base.html" %} -{# Copyright The IETF Trust 2015, All Rights Reserved #} {% load static %} {% load django_bootstrap5 %} {% block title %}Merge Persons{% endblock %} @@ -8,45 +8,17 @@

      Merge Person Records

      This tool will merge two Person records into one. If both records have logins and you want to retain the one on the left, use the Swap button to swap source and target records.

      - - {% if method == 'post' %} - {% csrf_token %} - {% endif %} +
      {% bootstrap_field form.source %} - {% if source %} - {% with person=source %} - {% include "person/person_info.html" %} - {% endwith %} - {% endif %}
      {% bootstrap_field form.target %} - {% if target %} - {% with person=target %} - {% include "person/person_info.html" %} - {% endwith %} - {% endif %}
      - {% if change_details %}{% endif %} - {% if warn_messages %} - {% for message in warn_messages %}{% endfor %} - {% endif %} - {% if method == 'post' %} - - Swap - - {% endif %} - {% endblock %} \ No newline at end of file diff --git a/ietf/templates/person/merge_request_email.txt b/ietf/templates/person/merge_request_email.txt new file mode 100644 index 0000000000..0a695f036c --- /dev/null +++ b/ietf/templates/person/merge_request_email.txt @@ -0,0 +1,23 @@ +Hello, + +We have identified multiple IETF Datatracker accounts that may represent a single person: + +https://datatracker.ietf.org/person/{{ source_account }} + +and + +https://datatracker.ietf.org/person/{{ target_account }} + +If this is so then it is important that we merge the accounts. + +This email is being sent to the primary emails associated with each Datatracker account. + +Please respond to this message individually from the email account(s) you control so we can take the appropriate action. + +If these should be merged, please identify which account you would like to keep the login credentials from. + +If you are associated with but no longer have access to one of the email accounts, then please let us know and we will follow up to determine how to proceed. + + +{{ sender_name }} +IETF Support \ No newline at end of file diff --git a/ietf/templates/person/merge_submit.html b/ietf/templates/person/merge_submit.html new file mode 100644 index 0000000000..30e1999f81 --- /dev/null +++ b/ietf/templates/person/merge_submit.html @@ -0,0 +1,57 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% extends "base.html" %} +{% load static %} +{% load django_bootstrap5 %} +{% block title %}Merge Persons{% endblock %} +{% block content %} +

      Merge Person Records

      +

      + This tool will merge two Person records into one. If both records have logins and you want to retain the one on the left, use the Swap button to swap source and target records. +

      + + {% csrf_token %} +
      +
      + {% bootstrap_field form.source %} + {% if source %} + {% with person=source %} + {% include "person/person_info.html" %} + {% endwith %} + {% endif %} +
      +
      + {% bootstrap_field form.target %} + {% if target %} + {% with person=target %} + {% include "person/person_info.html" %} + {% endwith %} + {% endif %} +
      +
      + {% if change_details %}{% endif %} + {% if warn_messages %} + {% for message in warn_messages %}{% endfor %} + {% endif %} + + + Swap + + + + + + Send Email + + + Back + + +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/person/person_link.html b/ietf/templates/person/person_link.html index f3f7e1a5b7..b77fe8d6df 100644 --- a/ietf/templates/person/person_link.html +++ b/ietf/templates/person/person_link.html @@ -1,7 +1,7 @@ {% if email and email == "system@datatracker.ietf.org" or name and name == "(System)" %}(System){% else %}{% if email or name %}{{ name }}{% if email and with_email %} {% if titlepage_name %}{{ titlepage_name }}{% else %}{{ name }}{% endif %}{% if email and with_email %} diff --git a/ietf/templates/person/profile.html b/ietf/templates/person/profile.html index 1424f037a1..a78a90412f 100644 --- a/ietf/templates/person/profile.html +++ b/ietf/templates/person/profile.html @@ -50,7 +50,11 @@

      Roles

       {% for role in person.role_set.all|active_roles %} - + {% for s in manual %} - {% if user.is_authenticated %} - - {% else %} - - {% endif %} +
      {{ reg.last_name }} {{ reg.first_name }} {{ reg.affiliation }} {{ reg.country_code }}{{ reg.reg_type }}{{ reg.attendance_type }}
      - + {{ session.group.acronym }} {% if session.purpose_id != "regular" and session.purpose_id != "none" %} diff --git a/ietf/secr/templates/sreq/session_approval_notification.txt b/ietf/templates/meeting/session_approval_notification.txt similarity index 56% rename from ietf/secr/templates/sreq/session_approval_notification.txt rename to ietf/templates/meeting/session_approval_notification.txt index 7bb63aa3fa..74eca09bd8 100644 --- a/ietf/secr/templates/sreq/session_approval_notification.txt +++ b/ietf/templates/meeting/session_approval_notification.txt @@ -1,3 +1,4 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} Dear {{ group.parent }} Director(s): {{ header }} meeting session request has just been @@ -5,11 +6,11 @@ submitted by {{ requester }}. The third session requires your approval. To approve the session go to the session request view here: -{{ settings.IDTRACKER_BASE_URL }}{% url "ietf.secr.sreq.views.view" acronym=group.acronym %} +{{ settings.IDTRACKER_BASE_URL }}{% url "ietf.meeting.views_session_request.view_request" acronym=group.acronym %} and click "Approve Third Session". Regards, The IETF Secretariat. -{% include "includes/session_info.txt" %} +{% include "meeting/session_request_info.txt" %} diff --git a/ietf/secr/templates/sreq/session_cancel_notification.txt b/ietf/templates/meeting/session_cancel_notification.txt similarity index 71% rename from ietf/secr/templates/sreq/session_cancel_notification.txt rename to ietf/templates/meeting/session_cancel_notification.txt index 8aee6c89db..3de67fc8f4 100644 --- a/ietf/secr/templates/sreq/session_cancel_notification.txt +++ b/ietf/templates/meeting/session_cancel_notification.txt @@ -1,3 +1,4 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} {% autoescape off %}{% load ams_filters %} A request to cancel a meeting session has just been submitted by {{ requester }}.{% endautoescape %} diff --git a/ietf/templates/meeting/session_details.html b/ietf/templates/meeting/session_details.html index 55fa3d3857..a4d9ba1090 100644 --- a/ietf/templates/meeting/session_details.html +++ b/ietf/templates/meeting/session_details.html @@ -1,5 +1,5 @@ {% extends "base.html" %} -{# Copyright The IETF Trust 2015, All Rights Reserved #} +{# Copyright The IETF Trust 2015-2026, All Rights Reserved #} {% load origin ietf_filters static %} {% block title %}{{ meeting }} : {{ group.acronym }}{% endblock %} {% block morecss %} @@ -53,69 +53,36 @@

      Unscheduled Sessions

      {% endif %} {% if forloop.last %}{% endif %} {% endfor %} + {% if user|has_role:"Secretariat" %} +
      +
      + Secretariat Only +
      +
      +
      + {% csrf_token %} + +
      +
      +
      + {% endif %} + {% comment %} + The existence of an element with id canManageMaterialsFlag is checked in + session_details.js to determine whether it should init the sortable tables. + Not the most elegant approach, but it works. + {% endcomment %} + {% if can_manage_materials %}
      {% endif %} {% endblock %} {% block js %} - {% if can_manage_materials %} - {% endif %} + {% endblock %} \ No newline at end of file diff --git a/ietf/templates/meeting/session_details_form.html b/ietf/templates/meeting/session_details_form.html index 6b59e7dacd..9cd1b6e85c 100644 --- a/ietf/templates/meeting/session_details_form.html +++ b/ietf/templates/meeting/session_details_form.html @@ -1,42 +1,48 @@ -{# Copyright The IETF Trust 2007-2020, All Rights Reserved #} +{# Copyright The IETF Trust 2007-2025, All Rights Reserved #} +{% load django_bootstrap5 %} +
      {% if hidden %} {{ form.name.as_hidden }}{{ form.purpose.as_hidden }}{{ form.type.as_hidden }}{{ form.requested_duration.as_hidden }} {{ form.has_onsite_tool.as_hidden }} {% else %} - - {% comment %} The form-group class is used by session_details_form.js to identify the correct element to hide the name / purpose / type fields when not needed. This is a bootstrap class - the secr app does not use it, so this (and the hidden class, also needed by session_details_form.js) are defined in edit.html and new.html as a kludge to make this work. {% endcomment %} - - - - - - - - - - - - - {% if not hide_onsite_tool_prompt %} - - - - - {% endif %} - -
      {{ form.name.label_tag }}{{ form.name }}{{ form.purpose.errors }}
      {{ form.purpose.label_tag }} - {{ form.purpose }}
      {{ form.type }}
      - {{ form.purpose.errors }}{{ form.type.errors }} -
      {{ form.requested_duration.label_tag }}{{ form.requested_duration }}{{ form.requested_duration.errors }}
      {{ form.has_onsite_tool.label_tag }}{{ form.has_onsite_tool }}{{ form.has_onsite_tool.errors }}
      - {% if hide_onsite_tool_prompt %}{{ form.has_onsite_tool.as_hidden }}{% endif %} + +
      + {% bootstrap_field form.name layout="horizontal" %} +
      + +
      +
      + +
      {{ form.purpose }}
      +
      {{ form.type }}
      + {{ form.purpose.errors }}{{ form.type.errors }} +
      +
      + + {% bootstrap_field form.requested_duration layout="horizontal" %} + {% if not hide_onsite_tool_prompt %} + {% bootstrap_field form.has_onsite_tool layout="horizontal" %} + {% endif %} + + {% if hide_onsite_tool_prompt %} + {{ form.has_onsite_tool.as_hidden }} + {% endif %} {% endif %} + {# hidden fields included whether or not the whole form is hidden #} - {{ form.attendees.as_hidden }}{{ form.comments.as_hidden }}{{ form.id.as_hidden }}{{ form.on_agenda.as_hidden }}{{ form.DELETE.as_hidden }}{{ form.remote_instructions.as_hidden }}{{ form.short.as_hidden }}{{ form.agenda_note.as_hidden }} -
      \ No newline at end of file + {{ form.attendees.as_hidden }} + {{ form.comments.as_hidden }} + {{ form.id.as_hidden }} + {{ form.on_agenda.as_hidden }} + {{ form.DELETE.as_hidden }} + {{ form.remote_instructions.as_hidden }} + {{ form.short.as_hidden }} + {{ form.agenda_note.as_hidden }} + diff --git a/ietf/secr/templates/sreq/not_meeting_notification.txt b/ietf/templates/meeting/session_not_meeting_notification.txt similarity index 83% rename from ietf/secr/templates/sreq/not_meeting_notification.txt rename to ietf/templates/meeting/session_not_meeting_notification.txt index 1120f8480c..0e5c940708 100644 --- a/ietf/secr/templates/sreq/not_meeting_notification.txt +++ b/ietf/templates/meeting/session_not_meeting_notification.txt @@ -1,3 +1,4 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} {% load ams_filters %} {{ login|smart_login }} {{ group.acronym }} working group, indicated that the {{ group.acronym }} working group does not plan to hold a session at IETF {{ meeting.number }}. diff --git a/ietf/templates/meeting/session_request_confirm.html b/ietf/templates/meeting/session_request_confirm.html new file mode 100644 index 0000000000..09043d3d0c --- /dev/null +++ b/ietf/templates/meeting/session_request_confirm.html @@ -0,0 +1,38 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% extends "base.html" %} +{% load static ietf_filters django_bootstrap5 %} +{% block title %}Confirm Session Request{% endblock %} + +{% block content %} +

      Confirm Session Request - IETF {{ meeting.number }}

      + +
      + Instructions +
      + +
      + +
      + + {% include "meeting/session_request_view_table.html" %} + +
      + {% csrf_token %} + {{ form }} + {{ form.session_forms.management_form }} + {% for sf in form.session_forms %} + {% include 'meeting/session_details_form.html' with form=sf hidden=True only %} + {% endfor %} + + + + +
      + +
      + +{% endblock %} + +{% block js %} + +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/meeting/session_request_form.html b/ietf/templates/meeting/session_request_form.html new file mode 100644 index 0000000000..ecf5cb7268 --- /dev/null +++ b/ietf/templates/meeting/session_request_form.html @@ -0,0 +1,206 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% extends "base.html" %} +{% load static ietf_filters django_bootstrap5 %} +{% block title %}{% if is_create %}New {% else %}Edit {% endif %}Session Request{% endblock %} +{% block morecss %}{{ block.super }} + .hidden {display: none !important;} + div.form-group {display: inline;} +{% endblock %} +{% block content %} +

      {% if is_create %}New {% else %}Edit {% endif %}Session Request

      + + {% if is_create %} +
      + Send a notification that the group does not plan to hold a session at IETF {{ meeting.number }} + Retrieve all information from previous meeting +
      + {% endif %} + +
      + +
      + {% csrf_token %} + {{ form.session_forms.management_form }} + {% if form.non_field_errors %} +
      {{ form.non_field_errors }}
      + {% endif %} + +
      + +
      + +
      +
      + +
      + +
      + +
      +
      + + {% bootstrap_field form.num_session layout="horizontal" %} + + {% if group.features.acts_like_wg %} + +
      +
      Session 1
      +
      + {% include 'meeting/session_details_form.html' with form=form.session_forms.0 hide_onsite_tool_prompt=True only %} +
      +
      + +
      +
      Session 2
      +
      + {% include 'meeting/session_details_form.html' with form=form.session_forms.1 hide_onsite_tool_prompt=True only %} +
      +
      + + {% if not is_virtual %} + {% bootstrap_field form.session_time_relation layout="horizontal" %} + {% endif %} + +
      +
      Additional Session Request
      +
      +
      + {{ form.third_session }} + +
      Additional slot may be available after agenda scheduling has closed and with the approval of an Area Director.
      +
      + +
      +
      + +
      +
      Third session request
      +
      + {% include 'meeting/session_details_form.html' with form=form.session_forms.2 hide_onsite_tool_prompt=True only %} +
      +
      + + {% else %}{# else not group.features.acts_like_wg #} + {% for session_form in form.session_forms %} +
      +
      Session {{ forloop.counter }}
      +
      + {% include 'meeting/session_details_form.html' with form=session_form only %} +
      +
      + {% endfor %} + {% endif %} + + {% bootstrap_field form.attendees layout="horizontal" %} + + {% bootstrap_field form.bethere layout="horizontal" %} + +
      +
      Conflicts to avoid
      +
      +
      +
      Other WGs that included {{ group.acronym }} in their conflict lists
      +
      {{ session_conflicts.inbound|default:"None" }}
      +
      +
      +
      WG Sessions
      You may select multiple WGs within each category
      +
      + {% for cname, cfield, cselector in form.wg_constraint_fields %} +
      +
      +
      +
      +
      + {{ cselector }} +
      +
      + +
      +
      +
      +
      + {{ cfield.errors }}{{ cfield }} +
      +
      +
      +
      + {% empty %}{# shown if there are no constraint fields #} +
      +
      No constraints are enabled for this meeting.
      + {% endfor %} +
      +
      + + {% if form.inactive_wg_constraint_count %} +
      +
      Disabled for this meeting
      +
      + {% for cname, value, field in form.inactive_wg_constraints %} +
      +
      {{ cname|title }}
      +
      +
      +
      + +
      +
      + + +
      +
      +
      +
      + {% endfor %} +
      +
      + {% endif %} + +
      +
      BOF Sessions
      +
      If the sessions can not be found in the fields above, please enter free form requests in the Special Requests field below.
      +
      +
      +
      + + {% if not is_virtual %} + + {% bootstrap_field form.resources layout="horizontal" %} + + {% bootstrap_field form.timeranges layout="horizontal" %} + + {% bootstrap_field form.adjacent_with_wg layout="horizontal" %} + +
      +
      Joint session with: (To request one session for multiple WGs together)
      +
      To request a joint session with another group, please contact the secretariat.
      +
      + + {% endif %} + + {% bootstrap_field form.comments layout="horizontal" %} + + {% if form.notifications_optional %} +
      + +
      +
      + + +
      +
      +
      + {% endif %} + + + Cancel +
      + +{% endblock %} +{% block js %} + + {{ form.media }} +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/meeting/session_request_info.txt b/ietf/templates/meeting/session_request_info.txt new file mode 100644 index 0000000000..2e96efb31f --- /dev/null +++ b/ietf/templates/meeting/session_request_info.txt @@ -0,0 +1,26 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% load ams_filters %} +--------------------------------------------------------- +Working Group Name: {{ group.name }} +Area Name: {{ group.parent }} +Session Requester: {{ login }} +{% if session.joint_with_groups %}{{ session.joint_for_session_display }} joint with: {{ session.joint_with_groups }}{% endif %} + +Number of Sessions: {{ session.num_session }} +Length of Session(s): {% for session_length in session_lengths %}{{ session_length.total_seconds|display_duration }}{% if not forloop.last %}, {% endif %}{% endfor %} +Number of Attendees: {{ session.attendees }} +Conflicts to Avoid: +{% for line in session.outbound_conflicts %} {{line}} +{% endfor %}{% if session.session_time_relation_display %} {{ session.session_time_relation_display }}{% endif %} +{% if session.adjacent_with_wg %} Adjacent with WG: {{ session.adjacent_with_wg }}{% endif %} +{% if session.timeranges_display %} Can't meet: {{ session.timeranges_display|join:", " }}{% endif %} + +Participants who must be present: +{% for person in session.bethere %} {{ person.ascii_name }} +{% endfor %} +Resources Requested: +{% for resource in session.resources %} {{ resource.desc }} +{% endfor %} +Special Requests: + {{ session.comments }} +--------------------------------------------------------- diff --git a/ietf/templates/meeting/session_request_list.html b/ietf/templates/meeting/session_request_list.html new file mode 100644 index 0000000000..789b7006e5 --- /dev/null +++ b/ietf/templates/meeting/session_request_list.html @@ -0,0 +1,65 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% extends "base.html" %} +{% load static %} +{% load ietf_filters %} +{% load django_bootstrap5 %} +{% block title %}Session Requests{% endblock %} +{% block content %} +

      Session Requests IETF {{ meeting.number }}

      + +
      + Instructions + + View list of timeslot requests + {% if user|has_role:"Secretariat" %} + {% if is_locked %} + Unlock Tool + {% else %} + Lock Tool + {% endif %} + {% endif %} +
      + +
      +
      + Request New Session +
      +
      +

      The list below includes those working groups that you currently chair which do not already have a session scheduled. You can click on an acronym to complete a request for a new session at the upcoming IETF meeting. Click "Group will not meet" to send a notification that the group does not plan to meet.

      +
        + {% for group in unscheduled_groups %} +
      • + {{ group.acronym }} + {% if group.not_meeting %} + (Currently, this group does not plan to hold a session at IETF {{ meeting.number }}) + {% endif %} +
        • + {% empty %} +
      • NONE
        • + {% endfor %} +
      +
      +
      + + +
      +
      + Edit / Cancel Previously Requested Sessions +
      +
      +

      The list below includes those working groups for which you or your co-chair has requested sessions at the upcoming IETF meeting. You can click on an acronym to initiate changes to a session, or cancel a session.

      + +
      +
      + +{% endblock %} + +{% block footer-extras %} + {% include "includes/sessions_footer.html" %} +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/meeting/session_request_locked.html b/ietf/templates/meeting/session_request_locked.html new file mode 100644 index 0000000000..15c023ce33 --- /dev/null +++ b/ietf/templates/meeting/session_request_locked.html @@ -0,0 +1,21 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% extends "base.html" %} +{% load static ietf_filters django_bootstrap5 %} +{% block title %}Session Request{% endblock %} + +{% block content %} +

      Session Request - IETF {{ meeting.number }}

      + + View list of timeslot requests + +
      + +
      +

      {{ message }}

      + +
      + +
      +
      + +{% endblock %} diff --git a/ietf/secr/templates/sreq/session_request_notification.txt b/ietf/templates/meeting/session_request_notification.txt similarity index 56% rename from ietf/secr/templates/sreq/session_request_notification.txt rename to ietf/templates/meeting/session_request_notification.txt index 75f2cbbae4..49dbbfc42c 100644 --- a/ietf/secr/templates/sreq/session_request_notification.txt +++ b/ietf/templates/meeting/session_request_notification.txt @@ -1,5 +1,6 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} {% autoescape off %}{% load ams_filters %} {% filter wordwrap:78 %}{{ header }} meeting session request has just been submitted by {{ requester }}.{% endfilter %} -{% include "includes/session_info.txt" %}{% endautoescape %} +{% include "meeting/session_request_info.txt" %}{% endautoescape %} diff --git a/ietf/templates/meeting/session_request_status.html b/ietf/templates/meeting/session_request_status.html new file mode 100644 index 0000000000..65e98d6d23 --- /dev/null +++ b/ietf/templates/meeting/session_request_status.html @@ -0,0 +1,28 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% extends "base.html" %} +{% load static %} +{% load ietf_filters %} +{% load django_bootstrap5 %} +{% block title %}Session Request Status{% endblock %} +{% block content %} +

      Session Request Status

      + +
      +
      + Session Request Status +
      +
      +

      Enter the message that you would like displayed to the WG Chair when this tool is locked.

      +
      {% csrf_token %} + {% bootstrap_form form %} + {% if is_locked %} + + {% else %} + + {% endif %} + +
      +
      +
      + +{% endblock %} diff --git a/ietf/templates/meeting/session_request_view.html b/ietf/templates/meeting/session_request_view.html new file mode 100644 index 0000000000..3db16f56cb --- /dev/null +++ b/ietf/templates/meeting/session_request_view.html @@ -0,0 +1,59 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% extends "base.html" %} +{% load static ietf_filters django_bootstrap5 %} +{% block title %}Session Request{% endblock %} + +{% block content %} +

      Session Request - IETF {{ meeting.number }}

      + +
      + Instructions +
      + +
      + +
      + + {% include "meeting/session_request_view_table.html" %} + +
      + +

      Activities Log

      +
      + + + + + + + + + + + {% for entry in activities %} + + + + + + + {% endfor %} + +
      DateTimeActionName
      {{ entry.act_date }}{{ entry.act_time }}{{ entry.activity }}{{ entry.act_by }}
      +
      + + + + {% if show_approve_button %} + Approve Third Session + {% endif %} + + Back + +
      + +{% endblock %} + +{% block js %} + +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/meeting/session_request_view_formset.html b/ietf/templates/meeting/session_request_view_formset.html new file mode 100644 index 0000000000..72811b8c2c --- /dev/null +++ b/ietf/templates/meeting/session_request_view_formset.html @@ -0,0 +1,49 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% load ams_filters %}{# keep this in sync with sessions_request_view_session_set.html #} +{% for sess_form in formset %} + {% if sess_form.cleaned_data and not sess_form.cleaned_data.DELETE %} +
      +
      + Session {{ forloop.counter }} +
      +
      +
      +
      Length
      +
      {{ sess_form.cleaned_data.requested_duration.total_seconds|display_duration }}
      +
      + {% if sess_form.cleaned_data.name %} +
      +
      Name
      +
      {{ sess_form.cleaned_data.name }}
      +
      + {% endif %} + {% if sess_form.cleaned_data.purpose.slug != 'regular' %} +
      +
      Purpose
      +
      + {{ sess_form.cleaned_data.purpose }} + {% if sess_form.cleaned_data.purpose.timeslot_types|length > 1 %}({{ sess_form.cleaned_data.type }} + ){% endif %} +
      +
      +
      +
      Onsite tool?
      +
      {{ sess_form.cleaned_data.has_onsite_tool|yesno }}
      +
      + {% endif %} +
      +
      + + {% if group.features.acts_like_wg and forloop.counter == 2 and not is_virtual %} +
      +
      + Time between sessions +
      +
      + {% if session.session_time_relation_display %}{{ session.session_time_relation_display }}{% else %}No + preference{% endif %} +
      +
      + {% endif %} + {% endif %} +{% endfor %} \ No newline at end of file diff --git a/ietf/templates/meeting/session_request_view_session_set.html b/ietf/templates/meeting/session_request_view_session_set.html new file mode 100644 index 0000000000..0b8412b04f --- /dev/null +++ b/ietf/templates/meeting/session_request_view_session_set.html @@ -0,0 +1,47 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% load ams_filters %}{# keep this in sync with sessions_request_view_formset.html #} +{% for sess in session_set %} +
      +
      + Session {{ forloop.counter }} +
      +
      +
      +
      Length
      +
      {{ sess.requested_duration.total_seconds|display_duration }}
      +
      + {% if sess.name %} +
      +
      Name
      +
      {{ sess.name }}
      +
      + {% endif %} + {% if sess.purpose.slug != 'regular' %} +
      +
      Purpose
      +
      + {{ sess.purpose }} + {% if sess.purpose.timeslot_types|length > 1 %}({{ sess.type }}){% endif %} +
      +
      +
      +
      Onsite tool?
      +
      {{ sess.has_onsite_tool|yesno }}
      +
      + {% endif %} +
      +
      + +{% if group.features.acts_like_wg and forloop.counter == 2 and not is_virtual %} +
      +
      + Time between sessions +
      +
      + {% if session.session_time_relation_display %}{{ session.session_time_relation_display }}{% else %}No + preference{% endif %} +
      +
      +{% endif %} + +{% endfor %} \ No newline at end of file diff --git a/ietf/templates/meeting/session_request_view_table.html b/ietf/templates/meeting/session_request_view_table.html new file mode 100644 index 0000000000..a5cb85c252 --- /dev/null +++ b/ietf/templates/meeting/session_request_view_table.html @@ -0,0 +1,146 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% load ams_filters %} + +
      +
      + Working Group Name +
      +
      + {{ group.name }} ({{ group.acronym }}) +
      +
      + +
      +
      + Area Name +
      +
      + {{ group.parent }} +
      +
      + +
      +
      + Number of Sessions Requested +
      +
      + {% if session.third_session %}3{% else %}{{ session.num_session }}{% endif %} +
      +
      + +{% if form %} + {% include 'meeting/session_request_view_formset.html' with formset=form.session_forms group=group session=session only %} +{% else %} + {% include 'meeting/session_request_view_session_set.html' with session_set=sessions group=group session=session only %} +{% endif %} + + +
      +
      + Number of Attendees +
      +
      + {{ session.attendees }} +
      +
      + +
      +
      + Conflicts to Avoid +
      +
      + {% if session_conflicts.outbound %} + {% for conflict in session_conflicts.outbound %} +
      +
      + {{ conflict.name|title }} +
      +
      + {{ conflict.groups }} +
      +
      + {% endfor %} + {% else %}None{% endif %} +
      +
      + +
      +
      + Other WGs that included {{ group }} in their conflict list +
      +
      + {% if session_conflicts.inbound %}{{ session_conflicts.inbound }}{% else %}None so far{% endif %} +
      +
      + +{% if not is_virtual %} +
      +
      + Resources requested +
      +
      + {% if session.resources %}
        {% for resource in session.resources %}
      • {{ resource.desc }}
        • {% endfor %}
      {% else %}None so far{% endif %} +
      +
      +{% endif %} + +
      +
      + Participants who must be present +
      +
      + {% if session.bethere %}
        {% for person in session.bethere %}
      • {{ person }}
        • {% endfor %}
      {% else %}None{% endif %} +
      +
      + +
      +
      + Can not meet on +
      +
      + {% if session.timeranges_display %}{{ session.timeranges_display|join:', ' }}{% else %}No constraints{% endif %} +
      +
      + +{% if not is_virtual %} +
      +
      + Adjacent with WG +
      +
      + {{ session.adjacent_with_wg|default:'No preference' }} +
      +
      +
      +
      + Joint session +
      +
      + {% if session.joint_with_groups %} + {{ session.joint_for_session_display }} with: {{ session.joint_with_groups }} + {% else %} + Not a joint session + {% endif %} +
      +
      +{% endif %} + +
      +
      + Special Requests +
      +
      + {{ session.comments }} +
      +
      + +{% if form and form.notifications_optional %} +
      +
      + {{ form.send_notifications.label}} +
      +
      + {% if form.cleaned_data.send_notifications %}Yes{% else %}No{% endif %} +
      +
      +{% endif %} diff --git a/ietf/templates/meeting/timeslot_edit.html b/ietf/templates/meeting/timeslot_edit.html index 11691ba6dd..3259dba9da 100644 --- a/ietf/templates/meeting/timeslot_edit.html +++ b/ietf/templates/meeting/timeslot_edit.html @@ -11,20 +11,22 @@ {% endcomment %} .timeslot-edit { overflow: auto; height: max(30rem, calc(100vh - 25rem));} .tstable { width: 100%; border-collapse: separate; } {# "separate" to ensure sticky cells keep their borders #} -.tstable thead { position: sticky; top: 0; z-index: 3; background-color: white;} -.tstable th:first-child, .tstable td:first-child { - background-color: white; {# needs to match the lighter of the striped-table colors! #} -position: sticky; -left: 0; - z-index: 2; {# render above other cells / borders but below thead (z-index 3, above) #} -} -.tstable tbody > tr:nth-of-type(odd) > th:first-child { - background-color: rgb(249, 249, 249); {# needs to match the darker of the striped-table colors! #} -} -.tstable th { white-space: nowrap;} -.tstable td { white-space: nowrap;} -.capacity { font-size:80%; font-weight: normal;} -a.new-timeslot-link { color: lightgray; font-size: large;} + .tstable tr th:first-child { min-width: 25rem; max-width: 25rem; overflow: hidden; } + .tstable thead { position: sticky; top: 0; z-index: 3; background-color: white;} + .tstable thead th span.day { position: sticky; left: 25.5rem; } + .tstable th:first-child, .tstable td:first-child { + background-color: white; {# needs to match the lighter of the striped-table colors! #} + position: sticky; + left: 0; + z-index: 2; {# render above other cells / borders but below thead (z-index 3, above) #} + } + .tstable tbody > tr:nth-of-type(odd) > th:first-child { + background-color: rgb(249, 249, 249); {# needs to match the darker of the striped-table colors! #} + } + .tstable th { white-space: nowrap;} + .tstable td { white-space: nowrap;} + .capacity { font-size:80%; font-weight: normal;} + a.new-timeslot-link { color: lightgray; font-size: large;} {% endblock %} {% block content %} {% origin %} @@ -84,12 +86,14 @@

      		- {{ day|date:'D' }} ({{ day }}) - - + + {{ day|date:'D' }} ({{ day }}) + + +
      {{ role.name.name }}{{ role.name.name }} + {% if role.name.name == 'Reviewer' %} + (See reviews) + {% endif %} + {{ role.group.name }} ({{ role.group.acronym }}) diff --git a/ietf/templates/person/send_merge_request.html b/ietf/templates/person/send_merge_request.html new file mode 100644 index 0000000000..f0c6272dca --- /dev/null +++ b/ietf/templates/person/send_merge_request.html @@ -0,0 +1,20 @@ +{# Copyright The IETF Trust 2025, All Rights Reserved #} +{% extends "base.html" %} +{% load static %} +{% load django_bootstrap5 %} +{% block title %}Send Merge Notice{% endblock %} +{% block content %} +

      Send Merge Notice

      + {% if form.non_field_errors %}
      {{ form.non_field_errors }}
      {% endif %} +
      + {% csrf_token %} + {% bootstrap_field form.to layout='horizontal' %} + {% bootstrap_field form.frm layout='horizontal' %} + {% bootstrap_field form.reply_to layout='horizontal' %} + {% bootstrap_field form.subject layout='horizontal' %} + {% bootstrap_field form.body layout='horizontal' %} + + Cancel +
      +{% endblock %} diff --git a/ietf/templates/registration/change_password.html b/ietf/templates/registration/change_password.html index 21c102bd0a..58bc2d2587 100644 --- a/ietf/templates/registration/change_password.html +++ b/ietf/templates/registration/change_password.html @@ -34,11 +34,14 @@

      Change password

      - Online attack: This password form uses the + Password strength requirements: + You must choose a password at least 12 characters long that scores at least a 3 according to the zxcvbn - password strength estimator to give an indication of password strength. - The crack time estimate given above assume online attack without rate - limiting, at a rate of 10 attempts per second. + password strength estimator. A warning will appear if your password does not meet this standard. +
      + Online attack: + The crack time estimate given above assumes an online attack at a rate of 10 attempts per second. + It is only a very rough guideline.
      Offline cracking: The datatracker currently uses the {{ hasher.algorithm }} diff --git a/ietf/templates/registration/edit_profile.html b/ietf/templates/registration/edit_profile.html index 1837016b15..1e4ab169e1 100644 --- a/ietf/templates/registration/edit_profile.html +++ b/ietf/templates/registration/edit_profile.html @@ -32,12 +32,19 @@

      Your account

      Change password
      - {% if person.photo %} + {% if person.photo or person.role_set.exists %}
      -
      {% include "person/photo.html" with person=person %}
      +
      + {% if person.photo %} + {% include "person/photo.html" with person=person %} + {% endif %} + {% if person.role_set.exists %} +

      Email support@ietf.org + to update your photo.

      + {% endif %}
      {% endif %}
      diff --git a/ietf/templates/stats/index.html b/ietf/templates/stats/index.html index 2000168525..38c8069507 100644 --- a/ietf/templates/stats/index.html +++ b/ietf/templates/stats/index.html @@ -14,8 +14,14 @@

      Reviews of Internet-Drafts in review teams (requires login) +
    • + Per country/affiliation registration for a specific meeting, by default IETF-{{ current_meeting }}. +
      • +
    • + Per country registration timeline for last meetings +

      • - Statistics on meetings and authorship are not currently available. + Statistics on authorship are not currently available.

      {% endblock %} \ No newline at end of file diff --git a/ietf/templates/stats/meeting_stats.html b/ietf/templates/stats/meeting_stats.html new file mode 100644 index 0000000000..fc41949a2e --- /dev/null +++ b/ietf/templates/stats/meeting_stats.html @@ -0,0 +1,58 @@ +{% extends "base.html" %} +{% load origin %} +{% origin %} +{% load ietf_filters static django_bootstrap5 %} +{% block js %} + {{ total_chart_data|json_script:"total-chart-data" }} + {{ in_person_chart_data|json_script:"in-person-chart-data" }} + +{% endblock %} +{% block content %} + {% origin %} +

      + {% block title %} + Statistics for IETF-{{ meeting_number }} ({{ meeting_date }}, {{ meeting_city }}, {{ meeting_country }}) Registrations + {% endblock %} +

      +
      + +
      + {% for slug, label, url in possible_stats_types %} + {{ label }} + {% endfor %} +
      + +
      + {% for num, url in possible_meeting_numbers %} + {{ num }} + {% endfor %} +
      +
      +

      + This page provides a visual representation of the total registrations for IETF-{{ meeting_number }} by {{ stats_type }}. + Only categories having more than {{ minimum_required }} registrations are displayed separately, + else they are grouped under "Other". +

      +
      +
      +

      Total Registrations by {{ stats_type|title }} ({{ total_total}} in total)

      +
      + +
      +
      +
      +

      In Person Registrations by {{ stats_type|title }} ({{ in_person_total}} in total)

      +
      + +
      +
      +
      +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/stats/meetings_timeline.html b/ietf/templates/stats/meetings_timeline.html new file mode 100644 index 0000000000..40f46880cc --- /dev/null +++ b/ietf/templates/stats/meetings_timeline.html @@ -0,0 +1,74 @@ +{% extends "base.html" %} +{% load origin %} +{% origin %} +{% load ietf_filters static django_bootstrap5 %} +{% block js %} + {{ total_chart_data|json_script:"total-chart-data" }} + {{ in_person_chart_data|json_script:"in-person-chart-data" }} + {{ stats_type|json_script:"stats-type-data" }} + +{% endblock %} +{% block content %} + {% origin %} +

      + {% block title %} + Statistics for IETF Meeting Registrations + {% endblock %} +

      + +
      + +
      + {% for slug, label, url in possible_stats_types %} + {{ label }} + {% endfor %} +
      + +
      + {% for num, url in possible_meeting_numbers %} + {{ num }} + {% endfor %} +
      +
      +

      + {% if stats_type == 'total' %} + This page provides a timeline of meeting registrations. + {% else %} + This page provides a timeline of meeting registrations by {{ stats_type }} with a limit of {{ top_n }} categories. + {% endif %} + Panning can be done via the mouse or with a finger. Zooming is done via the mouse wheel or via a pinch gesture. Press ESC + or click to reset panning/zooming. +

      +
      +
      + {% if stats_type == 'total' %} +

      Total Registrations

      + {% else %} +

      Total Registrations by {{ stats_type|title }}

      + {% endif %} +
      + +
      +
      + {% if stats_type != 'total' %} +
      + {% if stats_type == 'total' %} +

      Total In Person Registrations

      + {% else %} +

      In Person Registrations by {{ stats_type|title }}

      + {% endif %} +
      + +
      +
      + {% endif %} +
      +{% endblock %} \ No newline at end of file diff --git a/ietf/templates/submit/manual_post.html b/ietf/templates/submit/manual_post.html index 6e4a2ba42a..0da83e750f 100644 --- a/ietf/templates/submit/manual_post.html +++ b/ietf/templates/submit/manual_post.html @@ -1,5 +1,5 @@ {% extends "submit/submit_base.html" %} -{# Copyright The IETF Trust 2015, All Rights Reserved #} +{# Copyright The IETF Trust 2015-2026, All Rights Reserved #} {% load origin static %} {% block pagehead %} @@ -27,17 +27,9 @@

      Submissions needing manual posting

      - - {{ s.name }}-{{ s.rev }} - - - {{ s.name }}-{{ s.rev }} - + {{ s.name }}-{{ s.rev }} + {{ s.submission_date }} {% if s.passes_checks %} diff --git a/ietf/templates/submit/submit_base.html b/ietf/templates/submit/submit_base.html index 0613cf5d6a..a2c7be1a3b 100644 --- a/ietf/templates/submit/submit_base.html +++ b/ietf/templates/submit/submit_base.html @@ -4,7 +4,7 @@ {% block pagehead %}{{ block.super }}{% endblock %} {% block content %} {% origin %} -

      Internet-Draft submission

      +

      Submit an Internet-Draft

      • {% endblock %} diff --git a/ietf/templates/sync/bcp-index.txt b/ietf/templates/sync/bcp-index.txt new file mode 100644 index 0000000000..dd19920eba --- /dev/null +++ b/ietf/templates/sync/bcp-index.txt @@ -0,0 +1,52 @@ + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + BCP INDEX + ------------- + +(CREATED ON: {{created_on}}.) + +This file contains citations for all BCPs in numeric order. The BCPs +form a sub-series of the RFC document series, specifically those RFCs +with the status BEST CURRENT PRACTICE. + +BCP citations appear in this format: + + [BCP#] Best Current Practice #, + . + At the time of writing, this BCP comprises the following: + + Author 1, Author 2, "Title of the RFC", BCP #, RFC №, + DOI DOI string, Issue date, + . + +For example: + + [BCP3] Best Current Practice 3, + . + At the time of writing, this BCP comprises the following: + + F. Kastenholz, "Variance for The PPP Compression Control Protocol + and The PPP Encryption Control Protocol", BCP 3, RFC 1915, + DOI 10.17487/RFC1915, February 1996, + . + +Key to fields: + +# is the BCP number. + +№ is the RFC number. + +BCPs and other RFCs may be obtained from https://www.rfc-editor.org. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + BCP INDEX + --------- + + + +{% for bcp in bcps %}{{bcp|safe}} + +{% endfor %} diff --git a/ietf/templates/sync/fyi-index.txt b/ietf/templates/sync/fyi-index.txt new file mode 100644 index 0000000000..cf9d57d570 --- /dev/null +++ b/ietf/templates/sync/fyi-index.txt @@ -0,0 +1,52 @@ + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + FYI INDEX + ------------- + +(CREATED ON: {{created_on}}.) + +This file contains citations for all FYIs in numeric order. The FYIs +(For Your Information) documents form a sub-series of the RFC series, +specifically those documents that may be of particular interest +to Internet users. The corresponding RFCs have status INFORMATIONAL. + +FYI citations appear in this format: + + [FYI#] For Your Information #, + . + At the time of writing, this FYI comprises the following: + + Author 1, Author 2, "Title of the RFC", FYI #, RFC №, + DOI DOI string, Issue date, + . + +For example: + + [FYI8] For Your Information 8, + . + At the time of writing, this FYI comprises the following: + + B. Fraser, "Site Security Handbook", FYI 8, RFC 2196, + DOI 10.17487/RFC2196, September 1997, + . + +Key to fields: + +# is the FYI number. + +№ is the RFC number. + +FYIs and other RFCs may be obtained from https://www.rfc-editor.org. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + FYI INDEX + --------- + + + +{% for fyi in fyis %}{{fyi|safe}} + +{% endfor %} diff --git a/ietf/templates/sync/rfc-index.txt b/ietf/templates/sync/rfc-index.txt new file mode 100644 index 0000000000..0f01ddfa90 --- /dev/null +++ b/ietf/templates/sync/rfc-index.txt @@ -0,0 +1,69 @@ + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + RFC INDEX + ------------- + +(CREATED ON: {{created_on}}.) + +This file contains citations for all RFCs in numeric order. + +RFC citations appear in this format: + + #### Title of RFC. Author 1, Author 2, Author 3. Issue date. + (Format: ASCII) (Obsoletes xxx) (Obsoleted by xxx) (Updates xxx) + (Updated by xxx) (Also FYI ####) (Status: ssssss) (DOI: ddd) + +or + + #### Not Issued. + +For example: + + 1129 Internet Time Synchronization: The Network Time Protocol. D.L. + Mills. October 1989. (Format: TXT, PS, PDF, HTML) (Also RFC1119) + (Status: INFORMATIONAL) (DOI: 10.17487/RFC1129) + +Key to citations: + +#### is the RFC number. + +Following the RFC number are the title, the author(s), and the +publication date of the RFC. Each of these is terminated by a period. + +Following the number are the title (terminated with a period), the +author, or list of authors (terminated with a period), and the date +(terminated with a period). + +The format follows in parentheses. One or more of the following formats +are listed: text (TXT), PostScript (PS), Portable Document Format +(PDF), HTML, XML. + +Obsoletes xxxx refers to other RFCs that this one replaces; +Obsoleted by xxxx refers to RFCs that have replaced this one. +Updates xxxx refers to other RFCs that this one merely updates (but +does not replace); Updated by xxxx refers to RFCs that have updated +(but not replaced) this one. Generally, only immediately succeeding +and/or preceding RFCs are indicated, not the entire history of each +related earlier or later RFC in a related series. + +The (Also FYI ##) or (Also STD ##) or (Also BCP ##) phrase gives the +equivalent FYI, STD, or BCP number if the RFC is also in those +document sub-series. The Status field gives the document's +current status (see RFC 2026). The (DOI ddd) field gives the +Digital Object Identifier. + +RFCs may be obtained in a number of ways, using HTTP, FTP, or email. +See the RFC Editor Web page http://www.rfc-editor.org + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + RFC INDEX + --------- + + + +{% for rfc in rfcs %}{{rfc|safe}} + +{% endfor %} diff --git a/ietf/templates/sync/std-index.txt b/ietf/templates/sync/std-index.txt new file mode 100644 index 0000000000..a4a5fba946 --- /dev/null +++ b/ietf/templates/sync/std-index.txt @@ -0,0 +1,51 @@ + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + STD INDEX + ------------- + +(CREATED ON: {{created_on}}.) + +This file contains citations for all STDs in numeric order. Each +STD represents a single Internet Standard technical specification, +composed of one or more RFCs with Internet Standard status. + +STD citations appear in this format: + + [STD#] Internet Standard #, + . + At the time of writing, this STD comprises the following: + + Author 1, Author 2, "Title of the RFC", STD #, RFC №, + DOI DOI string, Issue date, + . + +For example: + + [STD6] Internet Standard 6, + . + At the time of writing, this STD comprises the following: + + J. Postel, "User Datagram Protocol", STD 6, RFC 768, + DOI 10.17487/RFC0768, August 1980, + . + +Key to fields: + +# is the STD number. + +№ is the RFC number. + +STDs and other RFCs may be obtained from https://www.rfc-editor.org. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + STD INDEX + --------- + + + +{% for std in stds %}{{std|safe}} + +{% endfor %} diff --git a/ietf/utils/admin.py b/ietf/utils/admin.py index 6c1c8726e1..cb8841cdc6 100644 --- a/ietf/utils/admin.py +++ b/ietf/utils/admin.py @@ -1,58 +1,30 @@ -# Copyright The IETF Trust 2011-2020, All Rights Reserved -# -*- coding: utf-8 -*- +# Copyright The IETF Trust 2011-2026, All Rights Reserved from django.contrib import admin -from django.utils.encoding import force_str - -def name(obj): - if hasattr(obj, 'abbrev'): - return obj.abbrev() - elif hasattr(obj, 'name'): - if callable(obj.name): - name = obj.name() - else: - name = force_str(obj.name) - if name: - return name - return str(obj) - -def admin_link(field, label=None, ordering="", display=name, suffix=""): - if not label: - label = field.capitalize().replace("_", " ").strip() - if ordering == "": - ordering = field - def _link(self): - obj = self - for attr in field.split("__"): - obj = getattr(obj, attr) - if callable(obj): - obj = obj() - if hasattr(obj, "all"): - objects = obj.all() - elif callable(obj): - objects = obj() - if not hasattr(objects, "__iter__"): - objects = [ objects ] - elif hasattr(obj, "__iter__"): - objects = obj - else: - objects = [ obj ] - chunks = [] - for obj in objects: - app = obj._meta.app_label - model = obj.__class__.__name__.lower() - id = obj.pk - chunks += [ '%(display)s' % - {'app':app, "model": model, "id":id, "display": display(obj), "suffix":suffix, } ] - return ", ".join(chunks) - _link.allow_tags = True - _link.short_description = label - _link.admin_order_field = ordering - return _link - -from .models import DumpInfo +from .models import DumpInfo, DirtyBits + + +class SaferStackedInline(admin.StackedInline): + """StackedInline without delete by default""" + + can_delete = False # no delete button + show_change_link = True # show a link to the resource (where it can be deleted) + + +class SaferTabularInline(admin.TabularInline): + """TabularInline without delete by default""" + + can_delete = False # no delete button + show_change_link = True # show a link to the resource (where it can be deleted) + + +@admin.register(DumpInfo) class DumpInfoAdmin(admin.ModelAdmin): - list_display = ['date', 'host', 'tz'] - list_filter = ['date'] -admin.site.register(DumpInfo, DumpInfoAdmin) + list_display = ["date", "host", "tz"] + list_filter = ["date"] + + +@admin.register(DirtyBits) +class DirtyBitsAdmin(admin.ModelAdmin): + list_display = ["slug", "dirty_time", "processed_time"] diff --git a/ietf/utils/aiosmtpd.py b/ietf/utils/aiosmtpd.py new file mode 100644 index 0000000000..3e4cd65dd9 --- /dev/null +++ b/ietf/utils/aiosmtpd.py @@ -0,0 +1,73 @@ +# Copyright The IETF Trust 2014-2025, All Rights Reserved +"""aiosmtpd-related utilities + +These are for testing / dev use. If you're using this for production code, think very +hard about the choices you're making... +""" +from aiosmtpd import handlers +from aiosmtpd.controller import Controller +from aiosmtpd.smtp import SMTP +from email.utils import parseaddr +from typing import Optional, TextIO + + +class SMTPTestHandler: + + def __init__(self, inbox: list): + self.inbox = inbox + + async def handle_DATA(self, server, session, envelope): + """Handle the DATA command and 'deliver' the message""" + + self.inbox.append(envelope.content) + # Per RFC2033: https://datatracker.ietf.org/doc/html/rfc2033.html#section-4.2 + # ...after the final ".", the server returns one reply + # for each previously successful RCPT command in the mail transaction, + # in the order that the RCPT commands were issued. Even if there were + # multiple successful RCPT commands giving the same forward-path, there + # must be one reply for each successful RCPT command. + return "\n".join("250 OK" for _ in envelope.rcpt_tos) + + async def handle_RCPT(self, server, session, envelope, address, rcpt_options): + """Handle an RCPT command and add the address to the envelope if it is acceptable""" + _, address = parseaddr(address) + if address == "": + return "501 Syntax: RCPT TO:
        " + if "poison" in address: + return "550 Error: Not touching that" + # At this point the address is acceptable + envelope.rcpt_tos.append(address) + return "250 OK" + + +class SMTPTestServerDriver: + + def __init__(self, address: str, port: int, inbox: Optional[list] = None): + # Allow longer lines than the 1001 that RFC 5321 requires. As of 2025-04-16 the + # datatracker emits some non-compliant messages. + # See https://aiosmtpd.aio-libs.org/en/latest/smtp.html + SMTP.line_length_limit = 4000 # tests start failing between 3000 and 4000 + self.controller = Controller( + hostname=address, + port=port, + handler=SMTPTestHandler(inbox=[] if inbox is None else inbox), + ) + + def start(self): + self.controller.start() + + def stop(self): + self.controller.stop() + + +class DevDebuggingHandler(handlers.Debugging): + """Debugging handler for use in dev ONLY""" + def __init__(self, stream: Optional[TextIO] = None): + # Allow longer lines than the 1001 that RFC 5321 requires. As of 2025-04-16 the + # datatracker emits some non-compliant messages. + # See https://aiosmtpd.aio-libs.org/en/latest/smtp.html + # Doing this in a handler class is a huge hack. Tests all pass with this set + # to 4000, but make the limit longer for dev just in case. + SMTP.line_length_limit = 10000 + super().__init__(stream) + diff --git a/ietf/utils/coverage.py b/ietf/utils/coverage.py new file mode 100644 index 0000000000..bd205ce586 --- /dev/null +++ b/ietf/utils/coverage.py @@ -0,0 +1,90 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +from coverage import Coverage, CoverageData, FileReporter +from coverage.control import override_config as override_coverage_config +from coverage.results import Numbers +from coverage.report_core import get_analysis_to_report +from coverage.results import Analysis +from django.conf import settings + + +class CoverageManager: + checker: Coverage | None = None + started = False + + def start(self): + if settings.SERVER_MODE != "production" and not self.started: + self.checker = Coverage( + source=[settings.BASE_DIR], + cover_pylib=False, + omit=settings.TEST_CODE_COVERAGE_EXCLUDE_FILES, + ) + for exclude_regex in getattr( + settings, + "TEST_CODE_COVERAGE_EXCLUDE_LINES", + [], + ): + self.checker.exclude(exclude_regex) + self.checker.start() + self.started = True + + def stop(self): + if self.checker is not None: + self.checker.stop() + + def save(self): + if self.checker is not None: + self.checker.save() + + def report(self, include: list[str] | None = None): + if self.checker is None: + return None + reporter = CustomDictReporter() + with override_coverage_config( + self.checker, + report_include=include, + ): + return reporter.report(self.checker) + + +class CustomDictReporter: # pragma: no cover + total = Numbers() + + def report(self, coverage): + coverage_data = coverage.get_data() + coverage_data.set_query_contexts(None) + measured_files = {} + for file_reporter, analysis in get_analysis_to_report(coverage, None): + measured_files[file_reporter.relative_filename()] = self.report_one_file( + coverage_data, + analysis, + file_reporter, + ) + tot_numer, tot_denom = self.total.ratio_covered + return { + "coverage": 1 if tot_denom == 0 else tot_numer / tot_denom, + "covered": measured_files, + "format": 5, + } + + def report_one_file( + self, + coverage_data: CoverageData, + analysis: Analysis, + file_reporter: FileReporter, + ): + """Extract the relevant report data for a single file.""" + nums = analysis.numbers + self.total += nums + n_statements = nums.n_statements + numer, denom = nums.ratio_covered + fraction_covered = 1 if denom == 0 else numer / denom + missing_line_nums = sorted(analysis.missing) + # Extract missing lines from source files + source_lines = file_reporter.source().splitlines() + missing_lines = [source_lines[num - 1] for num in missing_line_nums] + return ( + n_statements, + fraction_covered, + missing_line_nums, + missing_lines, + ) diff --git a/ietf/utils/db.py b/ietf/utils/db.py index d451f6cfd8..49c89da13a 100644 --- a/ietf/utils/db.py +++ b/ietf/utils/db.py @@ -1,28 +1,67 @@ -# Copyright The IETF Trust 2021, All Rights Reserved -# -*- coding: utf-8 -*- - -# Taken from/inspired by -# https://stackoverflow.com/questions/55147169/django-admin-jsonfield-default-empty-dict-wont-save-in-admin -# -# JSONField should recognize {}, (), and [] as valid, non-empty JSON -# values. However, the base Field class excludes them +# Copyright The IETF Trust 2021-2025, All Rights Reserved + import jsonfield +from django.db import models + +from ietf.utils.fields import ( + IETFJSONField as FormIETFJSONField, + EmptyAwareJSONField as FormEmptyAwareJSONField, +) + + +class EmptyAwareJSONField(models.JSONField): + """JSONField that allows empty JSON values when model specifies empty=False + + Taken from/inspired by + https://stackoverflow.com/questions/55147169/django-admin-jsonfield-default-empty-dict-wont-save-in-admin + + JSONField should recognize {}, (), and [] as valid, non-empty JSON values. -from ietf.utils.fields import IETFJSONField as FormIETFJSONField + If customizing the formfield, the field must accept the `empty_values` argument. + """ + + def __init__( + self, + *args, + empty_values=FormEmptyAwareJSONField.empty_values, + accepted_empty_values=None, + **kwargs, + ): + if accepted_empty_values is None: + accepted_empty_values = [] + self.empty_values = [x for x in empty_values if x not in accepted_empty_values] + super().__init__(*args, **kwargs) + + def formfield(self, **kwargs): + defaults = { + "form_class": FormEmptyAwareJSONField, + "empty_values": self.empty_values, + } + defaults.update(kwargs) + return super().formfield(**defaults) -class IETFJSONField(jsonfield.JSONField): +class IETFJSONField(jsonfield.JSONField): # pragma: no cover + # Deprecated - use EmptyAwareJSONField instead (different base class requires a + # new field name) + # Remove this class when migrations are squashed and it is no longer referenced form_class = FormIETFJSONField - def __init__(self, *args, empty_values=FormIETFJSONField.empty_values, accepted_empty_values=None, **kwargs): + def __init__( + self, + *args, + empty_values=FormIETFJSONField.empty_values, + accepted_empty_values=None, + **kwargs, + ): if accepted_empty_values is None: accepted_empty_values = [] - self.empty_values = [x - for x in empty_values - if x not in accepted_empty_values] + self.empty_values = [x for x in empty_values if x not in accepted_empty_values] super().__init__(*args, **kwargs) def formfield(self, **kwargs): - if 'form_class' not in kwargs or issubclass(kwargs['form_class'], FormIETFJSONField): - kwargs.setdefault('empty_values', self.empty_values) + if "form_class" not in kwargs or issubclass( + kwargs["form_class"], FormIETFJSONField + ): + kwargs.setdefault("empty_values", self.empty_values) return super().formfield(**{**kwargs}) diff --git a/ietf/utils/decorators.py b/ietf/utils/decorators.py index 56c28c4b19..b50e0e7f96 100644 --- a/ietf/utils/decorators.py +++ b/ietf/utils/decorators.py @@ -15,21 +15,9 @@ import debug # pyflakes:ignore -from ietf.utils.test_runner import set_coverage_checking from ietf.person.models import Person, PersonalApiKey, PersonApiKeyEvent from ietf.utils import log -def skip_coverage(f): - @wraps(f) - def _wrapper(*args, **kwargs): - if settings.TEST_CODE_COVERAGE_CHECKER: - set_coverage_checking(False) - result = f(*args, **kwargs) - set_coverage_checking(True) - return result - else: - return f(*args, **kwargs) - return _wrapper def person_required(f): @wraps(f) @@ -48,7 +36,7 @@ def require_api_key(f): @wraps(f) def _wrapper(request, *args, **kwargs): def err(code, text): - return HttpResponse(text, status=code, content_type='text/plain') + return HttpResponse(text, status=code, content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}") # Check method and get hash if request.method == 'POST': hash = request.POST.get('apikey') diff --git a/ietf/utils/fields.py b/ietf/utils/fields.py index 3e6f56d45e..6e8765612f 100644 --- a/ietf/utils/fields.py +++ b/ietf/utils/fields.py @@ -1,12 +1,11 @@ -# Copyright The IETF Trust 2012-2020, All Rights Reserved +# Copyright The IETF Trust 2012-2025, All Rights Reserved # -*- coding: utf-8 -*- import datetime import json import re - -import jsonfield +from email.utils import parseaddr import debug # pyflakes:ignore @@ -18,6 +17,7 @@ from django.core.exceptions import ValidationError from django.utils.dateparse import parse_duration + class MultiEmailField(forms.Field): def to_python(self, value): "Normalize data to a list of strings." @@ -40,6 +40,25 @@ def validate(self, value): for email in value: validate_email(email) + +def validate_name_addr_email(value): + "Validate name-addr style email address" + name, addr = parseaddr(value) + if not addr: + raise ValidationError("Invalid email format.") + try: + validate_email(addr) # validate the actual address part + except ValidationError: + raise ValidationError("Invalid email address.") + + +class NameAddrEmailField(forms.CharField): + def validate(self, value): + "Check if value consists only of valid emails." + super().validate(value) + validate_name_addr_email(value) + + def yyyymmdd_to_strftime_format(fmt): translation_table = sorted([ ("yyyy", "%Y"), @@ -328,8 +347,21 @@ def has_changed(self, initial, data): return super().has_changed(initial, data) -class IETFJSONField(jsonfield.fields.forms.JSONField): - def __init__(self, *args, empty_values=jsonfield.fields.forms.JSONField.empty_values, +class IETFJSONField(forms.JSONField): # pragma: no cover + # Deprecated - use EmptyAwareJSONField instead + def __init__(self, *args, empty_values=forms.JSONField.empty_values, + accepted_empty_values=None, **kwargs): + if accepted_empty_values is None: + accepted_empty_values = [] + self.empty_values = [x + for x in empty_values + if x not in accepted_empty_values] + + super().__init__(*args, **kwargs) + + +class EmptyAwareJSONField(forms.JSONField): + def __init__(self, *args, empty_values=forms.JSONField.empty_values, accepted_empty_values=None, **kwargs): if accepted_empty_values is None: accepted_empty_values = [] diff --git a/ietf/utils/jstest.py b/ietf/utils/jstest.py index 215d78d65f..cf242fc4eb 100644 --- a/ietf/utils/jstest.py +++ b/ietf/utils/jstest.py @@ -3,6 +3,8 @@ import os +from django.conf import settings +from django.contrib.staticfiles.testing import StaticLiveServerTestCase from django.urls import reverse as urlreverse from unittest import skipIf @@ -21,7 +23,11 @@ from ietf.utils.pipe import pipe -from ietf.utils.test_runner import IetfLiveServerTestCase +from ietf.utils.test_runner import ( + set_template_coverage, + set_url_coverage, + load_and_run_fixtures, +) executable_name = 'geckodriver' code, out, err = pipe('{} --version'.format(executable_name)) @@ -49,17 +55,44 @@ def ifSeleniumEnabled(func): return skipIf(skip_selenium, skip_message)(func) -class IetfSeleniumTestCase(IetfLiveServerTestCase): +class IetfSeleniumTestCase(StaticLiveServerTestCase): # pragma: no cover login_view = 'ietf.ietfauth.views.login' + @classmethod + def setUpClass(cls): + set_template_coverage(False) + set_url_coverage(False) + super().setUpClass() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + set_template_coverage(True) + set_url_coverage(True) + def setUp(self): - super(IetfSeleniumTestCase, self).setUp() + super().setUp() + # LiveServerTestCase uses TransactionTestCase which seems to + # somehow interfere with the fixture loading process in + # IetfTestRunner when running multiple tests (the first test + # is fine, in the next ones the fixtures have been wiped) - + # this is no doubt solvable somehow, but until then we simply + # recreate them here + from ietf.person.models import Person + if not Person.objects.exists(): + load_and_run_fixtures(verbosity=0) + self.replaced_settings = dict() + if hasattr(settings, 'IDTRACKER_BASE_URL'): + self.replaced_settings['IDTRACKER_BASE_URL'] = settings.IDTRACKER_BASE_URL + settings.IDTRACKER_BASE_URL = self.live_server_url self.driver = start_web_driver() self.driver.set_window_size(1024,768) def tearDown(self): - super(IetfSeleniumTestCase, self).tearDown() self.driver.close() + for k, v in self.replaced_settings.items(): + setattr(settings, k, v) + super().tearDown() def absreverse(self,*args,**kwargs): return '%s%s'%(self.live_server_url, urlreverse(*args, **kwargs)) diff --git a/ietf/utils/management/tests.py b/ietf/utils/management/tests.py index e94c39354f..38be464c7f 100644 --- a/ietf/utils/management/tests.py +++ b/ietf/utils/management/tests.py @@ -1,7 +1,7 @@ # Copyright The IETF Trust 2013-2020, All Rights Reserved # -*- coding: utf-8 -*- -import mock +from unittest import mock from django.core.management import call_command, CommandError from django.test import override_settings @@ -12,7 +12,7 @@ from ietf.utils.test_utils import TestCase -@mock.patch.object(EmailOnFailureCommand, 'handle') +@mock.patch.object(EmailOnFailureCommand, 'handle', return_value=None) class EmailOnFailureCommandTests(TestCase): def test_calls_handle(self, handle_method): call_command(EmailOnFailureCommand()) diff --git a/ietf/utils/meetecho.py b/ietf/utils/meetecho.py index 0dbf75736a..943f3789ef 100644 --- a/ietf/utils/meetecho.py +++ b/ietf/utils/meetecho.py @@ -27,7 +27,7 @@ class MeetechoAPI: - timezone = datetime.timezone.utc + timezone = datetime.UTC def __init__( self, api_base: str, client_id: str, client_secret: str, request_timeout=3.01 @@ -504,12 +504,17 @@ def _should_send_update(self, session): if self.slides_notify_time < datetime.timedelta(0): return True # < 0 means "always" for a scheduled session else: - now = datetime.datetime.now(tz=datetime.timezone.utc) + now = datetime.datetime.now(tz=datetime.UTC) return (timeslot.time - self.slides_notify_time) < now < (timeslot.end_time() + self.slides_notify_time) def add(self, session: "Session", slides: "Document", order: int): + """Add a slide deck to the session + + Returns True if the update was sent, False if it was not sent because the + current time is outside the update window for the session. + """ if not self._should_send_update(session): - return + return False # Would like to confirm that session.presentations includes the slides Document, but we can't # (same problem regarding unsaved Documents discussed in the docstring) @@ -524,11 +529,16 @@ def add(self, session: "Session", slides: "Document", order: int): "order": order, } ) + return True def delete(self, session: "Session", slides: "Document"): - """Delete a slide deck from the session""" + """Delete a slide deck from the session + + Returns True if the update was sent, False if it was not sent because the + current time is outside the update window for the session. + """ if not self._should_send_update(session): - return + return False if session.presentations.filter(document=slides).exists(): # "order" problems are very likely to result if we delete slides that are actually still @@ -543,12 +553,17 @@ def delete(self, session: "Session", slides: "Document"): id=slides.pk, ) if session.presentations.filter(document__type_id="slides").exists(): - self.send_update(session) # adjust order to fill in the hole + self._send_update(session) # adjust order to fill in the hole + return True def revise(self, session: "Session", slides: "Document"): - """Replace existing deck with its current state""" + """Replace existing deck with its current state + + Returns True if the update was sent, False if it was not sent because the + current time is outside the update window for the session. + """ if not self._should_send_update(session): - return + return False sp = session.presentations.filter(document=slides).first() if sp is None: @@ -561,11 +576,13 @@ def revise(self, session: "Session", slides: "Document"): id=slides.pk, ) self.add(session, slides, order) # fill in the hole + return True - def send_update(self, session: "Session"): - if not self._should_send_update(session): - return - + def _send_update(self, session: "Session"): + """Notify of the current state of the session's slides (no time window check) + + This is a private helper - use send_update() (no leading underscore) instead. + """ self.api.update_slide_decks( wg_token=self.wg_token(session.group), session=str(session.pk), @@ -580,3 +597,14 @@ def send_update(self, session: "Session"): for deck in session.presentations.filter(document__type="slides") ] ) + + def send_update(self, session: "Session"): + """Notify of the current state of the session's slides + + Returns True if the update was sent, False if it was not sent because the + current time is outside the update window for the session. + """ + if not self._should_send_update(session): + return False + self._send_update(session) + return True diff --git a/ietf/utils/migrations/0003_dirtybits.py b/ietf/utils/migrations/0003_dirtybits.py new file mode 100644 index 0000000000..11f6ed09f6 --- /dev/null +++ b/ietf/utils/migrations/0003_dirtybits.py @@ -0,0 +1,37 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("utils", "0002_delete_versioninfo"), + ] + + operations = [ + migrations.CreateModel( + name="DirtyBits", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "slug", + models.CharField( + choices=[("rfcindex", "RFC Index")], max_length=40, unique=True + ), + ), + ("dirty_time", models.DateTimeField(blank=True, null=True)), + ("processed_time", models.DateTimeField(blank=True, null=True)), + ], + options={ + "verbose_name_plural": "dirty bits", + }, + ), + ] diff --git a/ietf/utils/migrations/0004_alter_dirtybits_slug.py b/ietf/utils/migrations/0004_alter_dirtybits_slug.py new file mode 100644 index 0000000000..e17ea6cadd --- /dev/null +++ b/ietf/utils/migrations/0004_alter_dirtybits_slug.py @@ -0,0 +1,21 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("utils", "0003_dirtybits"), + ] + + operations = [ + migrations.AlterField( + model_name="dirtybits", + name="slug", + field=models.CharField( + choices=[("rfcindex", "RFC Index"), ("errata", "Errata Tags")], + max_length=40, + unique=True, + ), + ), + ] diff --git a/ietf/utils/mime.py b/ietf/utils/mime.py index ab21cfe5c6..1f9b75b4df 100644 --- a/ietf/utils/mime.py +++ b/ietf/utils/mime.py @@ -5,6 +5,7 @@ import magic import re + def get_mime_type(content): # try to fixup encoding if hasattr(magic, "open"): @@ -13,15 +14,17 @@ def get_mime_type(content): filetype = m.buffer(content) else: m = magic.Magic() - m.cookie = magic.magic_open(magic.MAGIC_NONE | magic.MAGIC_MIME | magic.MAGIC_MIME_ENCODING) + m.cookie = magic.magic_open( + magic.MAGIC_NONE | magic.MAGIC_MIME | magic.MAGIC_MIME_ENCODING + ) magic.magic_load(m.cookie, None) filetype = m.from_buffer(content) # Work around silliness in libmagic on OpenSUSE 15.1 - filetype = filetype.replace('text/x-Algol68;', 'text/plain;') - if ';' in filetype and 'charset=' in filetype: - mimetype, charset = re.split('; *charset=', filetype) + filetype = filetype.replace("text/x-Algol68;", "text/plain;") + filetype = filetype.replace("application/vnd.hp-HPGL;", "text/plain;") + if ";" in filetype and "charset=" in filetype: + mimetype, charset = re.split("; *charset=", filetype) else: - mimetype = re.split(';', filetype)[0] - charset = 'utf-8' + mimetype = re.split(";", filetype)[0] + charset = "utf-8" return mimetype, charset - diff --git a/ietf/utils/models.py b/ietf/utils/models.py index 21af5766e9..64f7f253f2 100644 --- a/ietf/utils/models.py +++ b/ietf/utils/models.py @@ -1,14 +1,36 @@ -# Copyright The IETF Trust 2015-2020, All Rights Reserved +# Copyright The IETF Trust 2015-2026, All Rights Reserved import itertools from django.db import models + +class DirtyBits(models.Model): + """A weak semaphore mechanism for coordination with celery beat tasks + + Web workers will set the "dirty_time" value for a given dirtybit slug. + Celery workers will do work if "processed_time" < "dirty_time" and update + "processed_time". + """ + + class Slugs(models.TextChoices): + RFCINDEX = "rfcindex", "RFC Index" + ERRATA = "errata", "Errata Tags" + + # next line can become `...choices=Slugs)` when we get to Django 5.x + slug = models.CharField(max_length=40, blank=False, choices=Slugs.choices, unique=True) + dirty_time = models.DateTimeField(null=True, blank=True) + processed_time = models.DateTimeField(null=True, blank=True) + + class Meta: + verbose_name_plural = "dirty bits" + + class DumpInfo(models.Model): date = models.DateTimeField() host = models.CharField(max_length=128) tz = models.CharField(max_length=32, default='UTC') - + class ForeignKey(models.ForeignKey): "A local ForeignKey proxy which provides the on_delete value required under Django 2.0." def __init__(self, to, on_delete=models.CASCADE, **kwargs): diff --git a/ietf/utils/resources.py b/ietf/utils/resources.py index 1252cfef14..63206eb33a 100644 --- a/ietf/utils/resources.py +++ b/ietf/utils/resources.py @@ -1,6 +1,4 @@ -# Copyright The IETF Trust 2014-2019, All Rights Reserved -# -*- coding: utf-8 -*- -# Autogenerated by the mkresources management command 2014-11-13 05:39 +# Copyright The IETF Trust 2014-2026, All Rights Reserved from ietf.api import ModelResource @@ -12,7 +10,7 @@ from django.contrib.contenttypes.models import ContentType from ietf import api -from ietf.utils.models import DumpInfo +from ietf.utils.models import DirtyBits, DumpInfo class UserResource(ModelResource): @@ -43,3 +41,9 @@ class Meta: "host": ALL, } api.utils.register(DumpInfoResource()) + + +class DirtyBitsResource(ModelResource): + class Meta: + queryset = DirtyBits.objects.none() +api.utils.register(DirtyBitsResource()) diff --git a/ietf/utils/searchindex.py b/ietf/utils/searchindex.py new file mode 100644 index 0000000000..6a8f4529a8 --- /dev/null +++ b/ietf/utils/searchindex.py @@ -0,0 +1,424 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +"""Search indexing utilities""" + +import re +from itertools import batched +from math import floor +from typing import Iterable +from urllib.parse import urljoin + +import httpx # just for exceptions +import requests +import typesense +import typesense.exceptions +from django.conf import settings +from typesense.types.document import DocumentSchema + +from ietf.doc.models import Document, StoredObject +from ietf.doc.storage_utils import retrieve_str +from ietf.utils.log import log + +# Error classes that might succeed just by retrying a failed attempt. +# Must be a tuple for use with isinstance() +RETRYABLE_ERROR_CLASSES = ( + httpx.ConnectError, + httpx.ConnectTimeout, + typesense.exceptions.Timeout, + typesense.exceptions.ServerError, + typesense.exceptions.ServiceUnavailable, +) + +DEFAULT_SETTINGS = { + "TYPESENSE_API_URL": "", + "TYPESENSE_API_KEY": "", + "TYPESENSE_COLLECTION_NAME": "docs", + "TASK_RETRY_DELAY": 10, + "TASK_MAX_RETRIES": 12, +} + + +def get_settings(): + return DEFAULT_SETTINGS | getattr(settings, "SEARCHINDEX_CONFIG", {}) + + +def enabled(): + _settings = get_settings() + return _settings["TYPESENSE_API_URL"] != "" + + +def get_typesense_client() -> typesense.Client: + _settings = get_settings() + client = typesense.Client( + { + "api_key": _settings["TYPESENSE_API_KEY"], + "nodes": [_settings["TYPESENSE_API_URL"]], + } + ) + return client + + +def get_collection_name() -> str: + _settings = get_settings() + collection_name = _settings["TYPESENSE_COLLECTION_NAME"] + assert isinstance(collection_name, str) + return collection_name + + +def _sanitize_text(content: str): + """Sanitize content text for search + + Aggressively simplifies whitespace, removes most punctuation + """ + # REs (with approximate names) + RE_DOT_OR_BANG_SPACE = r"\. |! " # -> " " (space) + RE_COMMENT_OR_TOC_CRUD = r"<--|-->|--+|\+|\.\.+" # -> "" + RE_BRACKETED_REF = r"\[[a-zA-Z0-9 -]+\]" # -> "" + RE_DOTTED_NUMBERS = r"[0-9]+\.[0-9]+(\.[0-9]+)?" # -> "" + RE_MULTIPLE_WHITESPACE = r"\s+" # -> " " (space) + # Replacement values (for clarity of intent) + SPACE = " " + EMPTY = "" + # Sanitizing begins here, order is significant! + content = re.sub(RE_DOT_OR_BANG_SPACE, SPACE, content.strip()) + content = re.sub(RE_COMMENT_OR_TOC_CRUD, EMPTY, content) + content = re.sub(RE_BRACKETED_REF, EMPTY, content) + content = re.sub(RE_DOTTED_NUMBERS, EMPTY, content) + content = re.sub(RE_MULTIPLE_WHITESPACE, SPACE, content) + return content.strip() + + +def _sanitize_abstract(abstract: str): + """Sanitize abstract text for search + + Simplifies whitespace but mostly leaves text intact. Abstract text will be + displayed in search results, so a light touch is needed. + """ + abstract = abstract.strip() + abstract = re.sub("\r\n|\n\r|\r", "\n", abstract) # normalize on \n + abstract = "\n".join(line.strip() for line in abstract.split("\n")) # strip by line + return abstract + + +def typesense_doc_from_rfc(rfc: Document) -> DocumentSchema: + assert rfc.type_id == "rfc" + assert rfc.rfc_number is not None + assert rfc.pages is not None + + keywords: list[str] = rfc.keywords # help type checking + + subseries = rfc.part_of() + if len(subseries) > 1: + log( + f"RFC {rfc.rfc_number} is in multiple subseries. " + f"Indexing as {subseries[0].name}" + ) + subseries = subseries[0] if len(subseries) > 0 else None + obsoleted_by = rfc.related_that("obs") + is_obsoleted = len(obsoleted_by) > 0 + updated_by = rfc.related_that("updates") + is_updated = len(updated_by) > 0 + is_historic = rfc.std_level.slug == "hist" + + stored_txt = ( + StoredObject.objects.exclude_deleted() + .filter(store="rfc", doc_name=rfc.name, name__startswith="txt/") + .first() + ) + content = "" + if stored_txt is not None: + # Should be available in the blobdb, but be cautious... + try: + content = retrieve_str(kind=stored_txt.store, name=stored_txt.name) + except Exception as err: + log(f"Unable to retrieve {stored_txt} from storage: {err}") + + ts_document = { + "id": f"doc-{rfc.pk}", + "rfcNumber": rfc.rfc_number, + "rfc": str(rfc.rfc_number), + "filename": rfc.name, + "title": rfc.title, + "abstract": _sanitize_abstract(rfc.abstract), + "pages": rfc.pages, + "keywords": keywords, + "type": "rfc", + "state": [state.name for state in rfc.states.all()], + "status": {"slug": rfc.std_level.slug, "name": rfc.std_level.name}, + "date": floor(rfc.time.timestamp()), + "publicationDate": floor(rfc.pub_datetime().timestamp()), + "stream": {"slug": rfc.stream.slug, "name": rfc.stream.name}, + "authors": [ + {"name": rfc_author.titlepage_name, "affiliation": rfc_author.affiliation} + for rfc_author in rfc.rfcauthor_set.all() + ], + "flags": { + "hiddenDefault": is_obsoleted or is_historic, + "obsoleted": is_obsoleted, + "updated": is_updated, + }, + "obsoletedBy": [str(doc.rfc_number) for doc in obsoleted_by], + "updatedBy": [str(doc.rfc_number) for doc in updated_by], + "ranking": rfc.rfc_number, + } + if subseries is not None: + ts_document["subseries"] = { + "acronym": subseries.type.slug, + "number": int(subseries.name[len(subseries.type.slug):]), + "total": len(subseries.contains()), + } + if rfc.group is not None: + ts_document["group"] = { + "acronym": rfc.group.acronym, + "name": rfc.group.name, + "full": f"{rfc.group.acronym} - {rfc.group.name}", + } + if ( + rfc.group.parent is not None + and rfc.stream_id not in ["ise", "irtf", "iab"] # exclude editorial? + ): + ts_document["area"] = { + "acronym": rfc.group.parent.acronym, + "name": rfc.group.parent.name, + "full": f"{rfc.group.parent.acronym} - {rfc.group.parent.name}", + } + if rfc.ad is not None: + ts_document["adName"] = rfc.ad.name + if content != "": + ts_document["content"] = _sanitize_text(content) + return ts_document + + +def update_or_create_rfc_entry(rfc: Document): + """Update/create index entries for one RFC""" + ts_document = typesense_doc_from_rfc(rfc) + client = get_typesense_client() + client.collections[get_collection_name()].documents.upsert(ts_document) + + +def update_or_create_rfc_entries( + rfcs: Iterable[Document], batchsize: int | None = None +): + """Update/create index entries for RFCs in bulk + + If batchsize is set, computes index data in batches of batchsize and adds to the + index. Will make a total of (len(rfcs) // batchsize) + 1 API calls. + + N.b. that typesense has a server-side batch size that defaults to 40, which should + "almost never be changed from the default." This does not change that. Further, + the python client library's import_ method has a batch_size parameter that does + client-side batching. We don't use that, either. + """ + success_count = 0 + fail_count = 0 + client = get_typesense_client() + batches = [rfcs] if batchsize is None else batched(rfcs, batchsize) + for batch in batches: + tdoc_batch = [typesense_doc_from_rfc(rfc) for rfc in batch] + results = client.collections[get_collection_name()].documents.import_( + tdoc_batch, {"action": "upsert"} + ) + for tdoc, result in zip(tdoc_batch, results): + if result["success"]: + success_count += 1 + else: + fail_count += 1 + log(f"Failed to index RFC {tdoc['rfcNumber']}: {result['error']}") + log(f"Added {success_count} RFCs to the index, failed to add {fail_count}") + + +DOCS_SCHEMA = { + "enable_nested_fields": True, + "default_sorting_field": "ranking", + "fields": [ + # RFC number in integer form, for sorting asc/desc in search results + # Omit field for drafts + { + "name": "rfcNumber", + "type": "int32", + "facet": False, + "optional": True, + "sort": True, + }, + # RFC number in string form, for direct matching with ranking + # Omit field for drafts + {"name": "rfc", "type": "string", "facet": False, "optional": True}, + # For drafts that correspond to an RFC, insert the RFC number + # Omit field for rfcs or if not relevant + {"name": "ref", "type": "string", "facet": False, "optional": True}, + # Filename of the document (without the extension, e.g. "rfc1234" + # or "draft-ietf-abc-def-02") + {"name": "filename", "type": "string", "facet": False, "infix": True}, + # Title of the draft / rfc + {"name": "title", "type": "string", "facet": False}, + # Abstract of the draft / rfc + {"name": "abstract", "type": "string", "facet": False}, + # Number of pages + {"name": "pages", "type": "int32", "facet": False}, + # A list of search keywords if relevant, set to empty array otherwise + {"name": "keywords", "type": "string[]", "facet": True}, + # Type of the document + # Accepted values: "draft" or "rfc" + {"name": "type", "type": "string", "facet": True}, + # State(s) of the document (e.g. "Published", "Adopted by a WG", etc.) + # Use the full name, not the slug + {"name": "state", "type": "string[]", "facet": True, "optional": True}, + # Status (Standard Level Name) + # Object with properties "slug" and "name" + # e.g.: { slug: "std", "name": "Internet Standard" } + {"name": "status", "type": "object", "facet": True, "optional": True}, + # The subseries it is part of. (e.g. "BCP") + # Omit otherwise. + { + "name": "subseries.acronym", + "type": "string", + "facet": True, + "optional": True, + }, + # The subseries number it is part of. (e.g. 123) + # Omit otherwise. + { + "name": "subseries.number", + "type": "int32", + "facet": True, + "sort": True, + "optional": True, + }, + # The total of RFCs in the subseries + # Omit if not part of a subseries + { + "name": "subseries.total", + "type": "int32", + "facet": False, + "sort": False, + "optional": True, + }, + # Date of the document, in unix epoch seconds (can be negative for < 1970) + {"name": "date", "type": "int64", "facet": False}, + # Expiration date of the document, in unix epoch seconds (can be negative + # for < 1970). Omit field for RFCs + {"name": "expires", "type": "int64", "facet": False, "optional": True}, + # Publication date of the RFC, in unix epoch seconds (can be negative + # for < 1970). Omit field for drafts + { + "name": "publicationDate", + "type": "int64", + "facet": True, + "optional": True, + }, + # Working Group + # Object with properties "acronym", "name" and "full" + # e.g.: + # { + # "acronym": "ntp", + # "name": "Network Time Protocols", + # "full": "ntp - Network Time Protocols", + # } + {"name": "group", "type": "object", "facet": True, "optional": True}, + # Area + # Object with properties "acronym", "name" and "full" + # e.g.: + # { + # "acronym": "mpls", + # "name": "Multiprotocol Label Switching", + # "full": "mpls - Multiprotocol Label Switching", + # } + {"name": "area", "type": "object", "facet": True, "optional": True}, + # Stream + # Object with properties "slug" and "name" + # e.g.: { slug: "ietf", "name": "IETF" } + {"name": "stream", "type": "object", "facet": True, "optional": True}, + # List of authors + # Array of objects with properties "name" and "affiliation" + # e.g.: + # [ + # {"name": "John Doe", "affiliation": "ACME Inc."}, + # {"name": "Ada Lovelace", "affiliation": "Babbage Corps."}, + # ] + {"name": "authors", "type": "object[]", "facet": True, "optional": True}, + # Area Director Name (e.g. "Leonardo DaVinci") + {"name": "adName", "type": "string", "facet": True, "optional": True}, + # Whether the document should be hidden by default in search results or not. + {"name": "flags.hiddenDefault", "type": "bool", "facet": True}, + # Whether the document is obsoleted by another document or not. + {"name": "flags.obsoleted", "type": "bool", "facet": True}, + # Whether the document is updated by another document or not. + {"name": "flags.updated", "type": "bool", "facet": True}, + # List of documents that obsolete this document. + # Array of strings. Use RFC number for RFCs. (e.g. ["123", "456"]) + # Omit if none. Must be provided if "flags.obsoleted" is set to True. + { + "name": "obsoletedBy", + "type": "string[]", + "facet": False, + "optional": True, + }, + # List of documents that update this document. + # Array of strings. Use RFC number for RFCs. (e.g. ["123", "456"]) + # Omit if none. Must be provided if "flags.updated" is set to True. + {"name": "updatedBy", "type": "string[]", "facet": False, "optional": True}, + # Sanitized content of the document. + # Make sure to remove newlines, double whitespaces, symbols and tags. + { + "name": "content", + "type": "string", + "facet": False, + "optional": True, + "store": False, + }, + # Ranking value to use when no explicit sorting is used during search + # Set to the RFC number for RFCs and the revision number for drafts + # This ensures newer RFCs get listed first in the default search results + # (without a query) + {"name": "ranking", "type": "int32", "facet": False}, + ], +} + +SEARCH_PRESETS = { + "red": { + "collection": "docs", + "infix": "off,always,off,off,off,off,off,off", + "query_by": "rfc,filename,title,abstract,keywords,authors,group,area", + "query_by_weights": "127,50,50,20,20,5,2,1" + }, + "red-content": { + "collection": "docs", + "infix": "off,always,off,off", + "query_by": "rfc,filename,authors,content", + "query_by_weights": "127,50,5,1" + }, +} + + +def create_collection(): + collection_name = get_collection_name() + log(f"Creating '{collection_name}' collection") + client = get_typesense_client() + client.collections.create({"name": get_collection_name()} | DOCS_SCHEMA) + + +def delete_collection(): + collection_name = get_collection_name() + log(f"Deleting '{collection_name}' collection") + client = get_typesense_client() + try: + client.collections[collection_name].delete() + except typesense.exceptions.ObjectNotFound: + pass + + +def upsert_presets(): + # typesense-python does not support presets, so use requests + _settings = get_settings() + api_base = _settings["TYPESENSE_API_URL"] + api_key = _settings["TYPESENSE_API_KEY"] + for preset_name, payload in SEARCH_PRESETS.items(): + log(f"Upserting '{preset_name}' preset") + response = requests.put( + urljoin(api_base, f"/presets/{preset_name}"), + json={"value": payload}, + headers={ + "X-TYPESENSE-API-KEY": api_key, + }, + timeout=3, + ) + response.raise_for_status() diff --git a/ietf/utils/serialize.py b/ietf/utils/serialize.py index 342d211cf5..77f97942cb 100644 --- a/ietf/utils/serialize.py +++ b/ietf/utils/serialize.py @@ -16,7 +16,7 @@ def object_as_shallow_dict(obj): if isinstance(f, models.ManyToManyField): v = list(v.values_list("pk", flat=True)) elif isinstance(f, models.DateTimeField): - v = v.astimezone(datetime.timezone.utc).isoformat() + v = v.astimezone(datetime.UTC).isoformat() elif isinstance(f, models.DateField): v = v.strftime('%Y-%m-%d') diff --git a/ietf/utils/storage.py b/ietf/utils/storage.py index 42fcf884aa..bad5af5178 100644 --- a/ietf/utils/storage.py +++ b/ietf/utils/storage.py @@ -1,8 +1,12 @@ # Copyright The IETF Trust 2020-2025, All Rights Reserved """Django Storage classes""" +import datetime +from hashlib import sha384 from pathlib import Path +from typing import Optional from django.conf import settings +from django.core.files.base import File from django.core.files.storage import FileSystemStorage from ietf.doc.storage_utils import store_file from .log import log @@ -48,9 +52,44 @@ def save(self, name, content, max_length=None): store_file(self.kind, blob_name, f, allow_overwrite=True) except Exception as err: log(f"Blobstore Error: Failed to shadow {saved_name} at {self.kind}:{blob_name}: {repr(err)}") + if settings.SERVER_MODE == "development": + raise return saved_name # includes the path! def deconstruct(self): path, args, kwargs = super().deconstruct() kwargs["kind"] = "" # don't record "kind" in migrations return path, args, kwargs + + +class MetadataFile(File): + """File that includes metadata""" + + def __init__(self, file, name=None, mtime: Optional[datetime.datetime]=None, content_type=""): + super().__init__(file=file, name=name) + self.mtime = mtime + self.content_type = content_type + self._custom_metadata = None + + @property + def custom_metadata(self): + if self._custom_metadata is None: + self._custom_metadata = self._compute_custom_metadata() + return self._custom_metadata + + def _compute_custom_metadata(self): + try: + self.file.seek(0) + except AttributeError: # TODO-BLOBSTORE + raise NotImplementedError("cannot handle unseekable content") + content_bytes = self.file.read() + if not isinstance( + content_bytes, bytes + ): # TODO-BLOBSTORE: This is sketch-development only -remove before committing + raise Exception(f"Expected bytes - got {type(content_bytes)}") + self.file.seek(0) + return { + "len": f"{len(content_bytes)}", + "sha384": f"{sha384(content_bytes).hexdigest()}", + "mtime": None if self.mtime is None else self.mtime.isoformat(), + } diff --git a/ietf/utils/templatetags/tests.py b/ietf/utils/templatetags/tests.py index a93bf2d94d..859319be3d 100644 --- a/ietf/utils/templatetags/tests.py +++ b/ietf/utils/templatetags/tests.py @@ -3,6 +3,7 @@ from django.template import Context, Origin, Template from django.test import override_settings +from ietf.utils.templatetags.textfilters import linkify from ietf.utils.test_utils import TestCase import debug # pyflakes: ignore @@ -39,3 +40,68 @@ def test_origin_outside_base_dir(self): output = template.render(Context()) self.assertNotIn(component, output, 'Full path components should not be revealed in html') + + +class TextfiltersTests(TestCase): + def test_linkify(self): + # Cases with autoescape = True (the default) + self.assertEqual( + linkify("plain string"), + "plain string", + ) + self.assertEqual( + linkify("https://www.ietf.org"), + 'https://www.ietf.org', + ) + self.assertEqual( + linkify('IETF'), + ( + '<a href="https://www.ietf.org">IETF</a>' + ), + ) + self.assertEqual( + linkify("somebody@example.com"), + 'somebody@example.com', + ) + self.assertEqual( + linkify("Some Body "), + ( + 'Some Body <' + 'somebody@example.com>' + ), + ) + self.assertEqual( + linkify(""), + "<script>alert('h4x0r3d');</script>", + ) + + # Cases with autoescape = False (these are dangerous and assume the caller + # has sanitized already) + self.assertEqual( + linkify("plain string", autoescape=False), + "plain string", + ) + self.assertEqual( + linkify("https://www.ietf.org", autoescape=False), + 'https://www.ietf.org', + ) + self.assertEqual( + linkify('IETF', autoescape=False), + 'IETF', + ) + self.assertEqual( + linkify("somebody@example.com", autoescape=False), + 'somebody@example.com', + ) + # bleach.Linkifier translates the < -> < and > -> > on this one + self.assertEqual( + linkify("Some Body ", autoescape=False), + ( + 'Some Body <' + 'somebody@example.com>' + ), + ) + self.assertEqual( + linkify("", autoescape=False), + "", + ) diff --git a/ietf/utils/templatetags/textfilters.py b/ietf/utils/templatetags/textfilters.py index 3b240740e0..e3bfbe0c56 100644 --- a/ietf/utils/templatetags/textfilters.py +++ b/ietf/utils/templatetags/textfilters.py @@ -7,6 +7,7 @@ from django import template from django.conf import settings from django.template.defaultfilters import stringfilter +from django.utils.html import conditional_escape from django.utils.safestring import mark_safe import debug # pyflakes:ignore @@ -71,10 +72,13 @@ def texescape_filter(value): "A TeX escape filter" return texescape(value) -@register.filter +@register.filter(needs_autoescape=True) @stringfilter -def linkify(value): - text = mark_safe(_linkify(value)) +def linkify(value, autoescape=True): + if autoescape: + # Escape unless the input was already a SafeString + value = conditional_escape(value) + text = mark_safe(_linkify(value)) # _linkify is a safe operation return text @register.filter diff --git a/ietf/utils/test_runner.py b/ietf/utils/test_runner.py index a77377ffb5..a23416e87f 100644 --- a/ietf/utils/test_runner.py +++ b/ietf/utils/test_runner.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2009-2020, All Rights Reserved +# Copyright The IETF Trust 2009-2025, All Rights Reserved # -*- coding: utf-8 -*- # # Portion Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). @@ -48,20 +48,17 @@ import subprocess import tempfile import copy +from contextlib import contextmanager + import boto3 import botocore.config import factory.random import urllib3 import warnings -from fnmatch import fnmatch from typing import Callable, Optional from urllib.parse import urlencode -from coverage.report import Reporter -from coverage.results import Numbers -from coverage.misc import NotPython - import django from django.conf import settings from django.contrib.staticfiles.testing import StaticLiveServerTestCase @@ -73,7 +70,7 @@ from django.template.loaders.filesystem import Loader as BaseLoader from django.test.runner import DiscoverRunner from django.core.management import call_command -from django.urls import URLResolver # type: ignore +from django.urls import URLResolver, resolve, Resolver404 # type: ignore from django.template.backends.django import DjangoTemplates from django.template.backends.django import Template # type: ignore[attr-defined] from django.utils import timezone @@ -85,23 +82,47 @@ import ietf import ietf.utils.mail from ietf.utils.management.commands import pyflakes -from ietf.utils.test_smtpserver import SMTPTestServerDriver +from ietf.utils.aiosmtpd import SMTPTestServerDriver from ietf.utils.test_utils import TestCase from mypy_boto3_s3.service_resource import Bucket +class UrlCoverageWarning(UserWarning): + """Warning category for URL coverage-related warnings""" + # URLs for which we don't expect patterns to match + IGNORE_URLS = ( + "/_doesnotexist/", + "/sitemap.xml.", + ) + + +class UninterestingPatternWarning(UrlCoverageWarning): + """Warning category for unexpected URL match patterns + + These are common, caused by tests that hit a URL that is not selected for + coverage checking. The warning is in place to help with a putative future + review of whether we're selecting the right patterns to check for coverage. + """ + pass + + +# Configure warnings for reasonable output quantity +warnings.simplefilter("once", UrlCoverageWarning) +warnings.simplefilter("ignore", UninterestingPatternWarning) + + loaded_templates: set[str] = set() visited_urls: set[str] = set() test_database_name: Optional[str] = None old_destroy: Optional[Callable] = None old_create: Optional[Callable] = None -template_coverage_collection = None -code_coverage_collection = None -url_coverage_collection = None +template_coverage_collection = False +url_coverage_collection = False validation_settings = {"validate_html": None, "validate_html_harder": None, "show_logging": False} + def start_vnu_server(port=8888): "Start a vnu validation server on the indicated port" vnu = subprocess.Popen( @@ -414,8 +435,9 @@ def do_append(res, p0, p1, item): res.append((str(item.pattern), item)) return res + _all_templates = None -def get_template_paths(apps=None): +def get_template_paths(apps=None) -> list[str]: global _all_templates if not _all_templates: # TODO: Add app templates to the full list, if we are using @@ -424,25 +446,30 @@ def get_template_paths(apps=None): templatepaths = settings.TEMPLATES[0]['DIRS'] for templatepath in templatepaths: for dirpath, dirs, files in os.walk(templatepath): - if ".svn" in dirs: - dirs.remove(".svn") - relative_path = dirpath[len(templatepath)+1:] - for file in files: - ignore = False - for pattern in settings.TEST_TEMPLATE_IGNORE: - if fnmatch(file, pattern): - ignore = True - break - if ignore: - continue - if relative_path != "": - file = os.path.join(relative_path, file) - templates.add(file) - if apps: - templates = [ t for t in templates if t.split(os.path.sep)[0] in apps ] - _all_templates = templates + # glob against path from PROJECT_DIR + project_path = pathlib.Path( + dirpath.removeprefix(settings.PROJECT_DIR).lstrip("/") + ) + # label entries with name relative to templatepath + relative_path = pathlib.Path( + dirpath.removeprefix(templatepath).lstrip("/") + ) + if ( + apps + and len(relative_path.parts) > 0 + and relative_path.parts[0] not in apps + ): + continue # skip uninteresting apps + for filename in files: + file_path = project_path / filename + if not any( + file_path.match(pat) for pat in settings.TEST_TEMPLATE_IGNORE + ): + templates.add(relative_path / filename) + _all_templates = [str(t) for t in templates] return _all_templates + def save_test_results(failures, test_labels): # Record the test result in a file, in order to be able to check the # results and avoid re-running tests if we've already run them with OK @@ -458,50 +485,29 @@ def save_test_results(failures, test_labels): tfile.write("%s OK\n" % (timestr, )) tfile.close() -def set_coverage_checking(flag=True): + +def set_template_coverage(flag): global template_coverage_collection - global code_coverage_collection + orig = template_coverage_collection + template_coverage_collection = flag + return orig + + +def set_url_coverage(flag): global url_coverage_collection - if settings.SERVER_MODE == 'test': - if flag: - settings.TEST_CODE_COVERAGE_CHECKER.collector.resume() - template_coverage_collection = True - code_coverage_collection = True - url_coverage_collection = True - else: - settings.TEST_CODE_COVERAGE_CHECKER.collector.pause() - template_coverage_collection = False - code_coverage_collection = False - url_coverage_collection = False - -class CoverageReporter(Reporter): - def report(self): - self.find_file_reporters(None) - - total = Numbers() - result = {"coverage": 0.0, "covered": {}, "format": 5, } - for fr in self.file_reporters: - try: - analysis = self.coverage._analyze(fr) - nums = analysis.numbers - missing_nums = sorted(analysis.missing) - with io.open(analysis.filename, encoding='utf-8') as file: - lines = file.read().splitlines() - missing_lines = [ lines[l-1] for l in missing_nums ] - result["covered"][fr.relative_filename()] = (nums.n_statements, nums.pc_covered/100.0, missing_nums, missing_lines) - total += nums - except KeyboardInterrupt: # pragma: not covered - raise - except Exception: - report_it = not self.config.ignore_errors - if report_it: - typ, msg = sys.exc_info()[:2] - if typ is NotPython and not fr.should_be_python(): - report_it = False - if report_it: - raise - result["coverage"] = total.pc_covered/100.0 - return result + orig = url_coverage_collection + url_coverage_collection = flag + return orig + + +@contextmanager +def disable_coverage(): + """Context manager/decorator that disables template/url coverage""" + orig_template = set_template_coverage(False) + orig_url = set_url_coverage(False) + yield + set_template_coverage(orig_template) + set_url_coverage(orig_url) class CoverageTest(unittest.TestCase): @@ -568,44 +574,62 @@ def ignore_pattern(regex, pattern): ) or pattern.callback == django.views.static.serve) - patterns = [(regex, re.compile(regex, re.U), obj) for regex, obj in url_patterns - if not ignore_pattern(regex, obj)] + patterns ={ + regex: obj + for regex, obj in url_patterns + if not ignore_pattern(regex, obj) + } covered = set() for url in visited_urls: - for regex, compiled, obj in patterns: - if regex not in covered and compiled.match(url[1:]): # strip leading / - covered.add(regex) - break + try: + resolved = resolve(url) # let Django resolve the URL for us + except Resolver404: + if url not in UrlCoverageWarning.IGNORE_URLS: + warnings.warn( + f"Unable to resolve visited URL {url}", UrlCoverageWarning + ) + continue + if resolved.route not in patterns: + warnings.warn( + f"WARNING: url resolved to an unexpected pattern (url='{url}', " + f"resolved to r'{resolved.route}'", + UninterestingPatternWarning, + ) + continue + covered.add(resolved.route) self.runner.coverage_data["url"] = { - "coverage": 1.0*len(covered)/len(patterns), - "covered": dict( (k, (o.lookup_str, k in covered)) for k,p,o in patterns ), + "coverage": 1.0 * len(covered) / len(patterns), + "covered": dict( + (k, (o.lookup_str, k in covered)) for k, o in patterns.items() + ), "format": 4, - } + } self.report_test_result("url") else: self.skipTest("Coverage switched off with --skip-coverage") def code_coverage_test(self): - if self.runner.check_coverage: - include = [ os.path.join(path, '*') for path in self.runner.test_paths ] - checker = self.runner.code_coverage_checker - checker.stop() + if ( + self.runner.check_coverage + and settings.TEST_CODE_COVERAGE_CHECKER is not None + ): + coverage_manager = settings.TEST_CODE_COVERAGE_CHECKER + coverage_manager.stop() # Save to the .coverage file - checker.save() + coverage_manager.save() # Apply the configured and requested omit and include data - checker.config.from_args(ignore_errors=None, omit=settings.TEST_CODE_COVERAGE_EXCLUDE_FILES, - include=include, file=None) - for pattern in settings.TEST_CODE_COVERAGE_EXCLUDE_LINES: - checker.exclude(pattern) # Maybe output an HTML report if self.runner.run_full_test_suite and self.runner.html_report: - checker.html_report(directory=settings.TEST_CODE_COVERAGE_REPORT_DIR) - # In any case, build a dictionary with per-file data for this run - reporter = CoverageReporter(checker, checker.config) - self.runner.coverage_data["code"] = reporter.report() + coverage_manager.checker.html_report( + directory=settings.TEST_CODE_COVERAGE_REPORT_DIR + ) + # Generate the output report data + self.runner.coverage_data["code"] = coverage_manager.report( + include=[str(pathlib.Path(p) / "*") for p in self.runner.test_paths] + ) self.report_test_result("code") else: self.skipTest("Coverage switched off with --skip-coverage") @@ -819,23 +843,12 @@ def setup_test_environment(self, **kwargs): "covered": {}, "format": 1, }, - "migration": { - "present": {}, - "format": 3, - } } settings.TEMPLATES[0]['OPTIONS']['loaders'] = ('ietf.utils.test_runner.TemplateCoverageLoader',) + settings.TEMPLATES[0]['OPTIONS']['loaders'] settings.MIDDLEWARE = ('ietf.utils.test_runner.record_urls_middleware',) + tuple(settings.MIDDLEWARE) - self.code_coverage_checker = settings.TEST_CODE_COVERAGE_CHECKER - if not self.code_coverage_checker._started: - sys.stderr.write(" ** Warning: In %s: Expected the coverage checker to have\n" - " been started already, but it wasn't. Doing so now. Coverage numbers\n" - " will be off, though.\n" % __name__) - self.code_coverage_checker.start() - if settings.SITE_ID != 1: print(" Changing SITE_ID to '1' during testing.") settings.SITE_ID = 1 @@ -863,7 +876,7 @@ def setup_test_environment(self, **kwargs): try: # remember the value so ietf.utils.mail.send_smtp() will use the same ietf.utils.mail.SMTP_ADDR['port'] = base + offset - self.smtpd_driver = SMTPTestServerDriver((ietf.utils.mail.SMTP_ADDR['ip4'],ietf.utils.mail.SMTP_ADDR['port']),None) + self.smtpd_driver = SMTPTestServerDriver(ietf.utils.mail.SMTP_ADDR['ip4'],ietf.utils.mail.SMTP_ADDR['port'], None) self.smtpd_driver.start() print((" Running an SMTP test server on %(ip4)s:%(port)s to catch outgoing email." % ietf.utils.mail.SMTP_ADDR)) break @@ -1135,9 +1148,8 @@ def _extra_tests(self): ), ] if self.check_coverage: - global template_coverage_collection, code_coverage_collection, url_coverage_collection + global template_coverage_collection, url_coverage_collection template_coverage_collection = True - code_coverage_collection = True url_coverage_collection = True tests += [ PyFlakesTestCase(test_runner=self, methodName='pyflakes_test'), @@ -1221,37 +1233,6 @@ def run_tests(self, test_labels, extra_tests=None, **kwargs): return failures -class IetfLiveServerTestCase(StaticLiveServerTestCase): - @classmethod - def setUpClass(cls): - set_coverage_checking(False) - super(IetfLiveServerTestCase, cls).setUpClass() - - def setUp(self): - super(IetfLiveServerTestCase, self).setUp() - # LiveServerTestCase uses TransactionTestCase which seems to - # somehow interfere with the fixture loading process in - # IetfTestRunner when running multiple tests (the first test - # is fine, in the next ones the fixtures have been wiped) - - # this is no doubt solvable somehow, but until then we simply - # recreate them here - from ietf.person.models import Person - if not Person.objects.exists(): - load_and_run_fixtures(verbosity=0) - self.replaced_settings = dict() - if hasattr(settings, 'IDTRACKER_BASE_URL'): - self.replaced_settings['IDTRACKER_BASE_URL'] = settings.IDTRACKER_BASE_URL - settings.IDTRACKER_BASE_URL = self.live_server_url - - @classmethod - def tearDownClass(cls): - super(IetfLiveServerTestCase, cls).tearDownClass() - set_coverage_checking(True) - - def tearDown(self): - for k, v in self.replaced_settings.items(): - setattr(settings, k, v) - super().tearDown() class TestBlobstoreManager(): # N.B. buckets and blobstore are intentional Class-level attributes @@ -1262,13 +1243,17 @@ class TestBlobstoreManager(): aws_access_key_id="minio_root", aws_secret_access_key="minio_pass", aws_session_token=None, - config = botocore.config.Config(signature_version="s3v4"), + config = botocore.config.Config( + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + signature_version="s3v4", + ), #config=botocore.config.Config(signature_version=botocore.UNSIGNED), verify=False ) def createTestBlobstores(self): - for storagename in settings.MORE_STORAGE_NAMES: + for storagename in settings.ARTIFACT_STORAGE_NAMES: bucketname = f"test-{storagename}" try: bucket = self.blobstore.create_bucket(Bucket=bucketname) diff --git a/ietf/utils/test_smtpserver.py b/ietf/utils/test_smtpserver.py deleted file mode 100644 index 66675aa0b1..0000000000 --- a/ietf/utils/test_smtpserver.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright The IETF Trust 2014-2020, All Rights Reserved -# -*- coding: utf-8 -*- - - -import smtpd -import threading -import asyncore - -import debug # pyflakes:ignore - -class AsyncCoreLoopThread(object): - - def wrap_loop(self, exit_condition, timeout=1.0, use_poll=False, map=None): - if map is None: - map = asyncore.socket_map - while map and not exit_condition: - asyncore.loop(timeout=1.0, use_poll=False, map=map, count=1) - - def start(self): - """Start the listening service""" - self.exit_condition = [] - kwargs={'exit_condition':self.exit_condition,'timeout':1.0} - self.thread = threading.Thread(target=self.wrap_loop, kwargs=kwargs) - self.thread.daemon = True - self.thread.daemon = True - self.thread.start() - - def stop(self): - """Stop the listening service""" - self.exit_condition.append(True) - self.thread.join() - - -class SMTPTestChannel(smtpd.SMTPChannel): - -# mail_options = ['BODY=8BITMIME', 'SMTPUTF8'] - - def smtp_RCPT(self, arg): - if not self.mailfrom: - self.push(str('503 Error: need MAIL command')) - return - arg = self._strip_command_keyword('TO:', arg) - address, __ = self._getaddr(arg) - if not address: - self.push(str('501 Syntax: RCPT TO:
        ')) - return - if "poison" in address: - self.push(str('550 Error: Not touching that')) - return - self.rcpt_options = [] - self.rcpttos.append(address) - self.push(str('250 Ok')) - -class SMTPTestServer(smtpd.SMTPServer): - - def __init__(self,localaddr,remoteaddr,inbox): - if inbox is not None: - self.inbox=inbox - else: - self.inbox = [] - smtpd.SMTPServer.__init__(self,localaddr,remoteaddr) - - def handle_accept(self): - pair = self.accept() - if pair is not None: - conn, addr = pair - #channel = SMTPTestChannel(self, conn, addr) - SMTPTestChannel(self, conn, addr) - - def process_message(self, peer, mailfrom, rcpttos, data, mail_options=None, rcpt_options=None): - self.inbox.append(data) - - -class SMTPTestServerDriver(object): - def __init__(self, localaddr, remoteaddr, inbox=None): - self.localaddr=localaddr - self.remoteaddr=remoteaddr - if inbox is not None: - self.inbox = inbox - else: - self.inbox = [] - self.thread_driver = None - - def start(self): - self.smtpserver = SMTPTestServer(self.localaddr,self.remoteaddr,self.inbox) - self.thread_driver = AsyncCoreLoopThread() - self.thread_driver.start() - - def stop(self): - if self.thread_driver: - self.thread_driver.stop() - diff --git a/ietf/utils/test_utils.py b/ietf/utils/test_utils.py index 86c5a0c1c3..5faf83d93f 100644 --- a/ietf/utils/test_utils.py +++ b/ietf/utils/test_utils.py @@ -38,6 +38,7 @@ import re import email import html5lib +import rest_framework.test import requests_mock import shutil import sys @@ -312,3 +313,11 @@ def tearDown(self): shutil.rmtree(dir) self.requests_mock.stop() super().tearDown() + + +class APITestCase(TestCase): + """Test case that uses rest_framework's APIClient + + This is equivalent to rest_framework.test.APITestCase, but picks up our + """ + client_class = rest_framework.test.APIClient diff --git a/ietf/utils/tests.py b/ietf/utils/tests.py index 2dd861cd11..99c33f34b3 100644 --- a/ietf/utils/tests.py +++ b/ietf/utils/tests.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2014-2020, All Rights Reserved +# Copyright The IETF Trust 2014-2025, All Rights Reserved # -*- coding: utf-8 -*- @@ -11,7 +11,7 @@ import shutil import types -from mock import call, patch +from unittest.mock import call, patch from pyquery import PyQuery from typing import Dict, List # pyflakes:ignore @@ -19,7 +19,6 @@ from email.mime.image import MIMEImage from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText -from fnmatch import fnmatch from importlib import import_module from textwrap import dedent from tempfile import mkdtemp @@ -55,9 +54,12 @@ decode_header_value, show_that_mail_was_sent, ) -from ietf.utils.test_runner import get_template_paths, set_coverage_checking +from ietf.utils.test_runner import ( + get_template_paths, + set_template_coverage, + set_url_coverage, +) from ietf.utils.test_utils import TestCase, unicontent -from ietf.utils.text import parse_unicode from ietf.utils.timezone import timezone_not_near_midnight from ietf.utils.xmldraft import XMLDraft, InvalidMetadataError, capture_xml2rfc_output @@ -312,15 +314,16 @@ def qualified(name): return list(callbacks) -class TemplateChecksTestCase(TestCase): +class TemplateChecksTestCase(TestCase): # pragma: no cover paths = [] # type: List[str] templates = {} # type: Dict[str, Template] def setUp(self): super().setUp() - set_coverage_checking(False) - self.paths = list(get_template_paths()) + set_template_coverage(False) + set_url_coverage(False) + self.paths = get_template_paths() # already filtered ignores self.paths.sort() for path in self.paths: try: @@ -329,17 +332,14 @@ def setUp(self): pass def tearDown(self): - set_coverage_checking(True) + set_template_coverage(True) + set_url_coverage(True) super().tearDown() def test_parse_templates(self): errors = [] for path in self.paths: - for pattern in settings.TEST_TEMPLATE_IGNORE: - if fnmatch(path, pattern): - continue - if not path in self.templates: - + if path not in self.templates: try: get_template(path) except Exception as e: @@ -709,6 +709,14 @@ def test_render_author_name(self): )), "Joanna Q. Public", ) + self.assertEqual( + XMLDraft.render_author_name(lxml.etree.Element( + "author", + fullname=chr(340)+"ich", + asciiFullname="Rich UTF-8", + )), + chr(340)+"ich (Rich UTF-8)", + ) self.assertEqual( XMLDraft.render_author_name(lxml.etree.Element( "author", @@ -741,6 +749,23 @@ def test_render_author_name(self): "J. Q.", ) + @patch("ietf.utils.xmldraft.XMLDraft.__init__", return_value=None) + def test_get_title(self, mock_init): + xmldraft = XMLDraft("fake") + self.assertTrue(mock_init.called) + # Stub XML that does not have a front/title element + xmldraft.xmlroot = lxml.etree.XML( + "" # no title + ) + self.assertEqual(xmldraft.get_title(), "") + + # Stub XML that has a front/title element + xmldraft.xmlroot = lxml.etree.XML( + "This Is the Title" + ) + self.assertEqual(xmldraft.get_title(), "This Is the Title") + + def test_capture_xml2rfc_output(self): """capture_xml2rfc_output reroutes and captures xml2rfc logs""" orig_write_out = xml2rfc_log.write_out @@ -838,24 +863,6 @@ def test_assertion(self): assertion('False') settings.SERVER_MODE = 'test' -class TestRFC2047Strings(TestCase): - def test_parse_unicode(self): - names = ( - ('=?utf-8?b?4Yuz4YuK4Ym1IOGJoOGJgOGIiA==?=', 'ዳዊት በቀለ'), - ('=?utf-8?b?5Li9IOmDnA==?=', '丽 郜'), - ('=?utf-8?b?4KSV4KSu4KWN4KSs4KWL4KScIOCkoeCkvuCksA==?=', 'कम्बोज डार'), - ('=?utf-8?b?zpfPgc6szrrOu861zrnOsSDOm865z4zOvc+Ezrc=?=', 'Ηράκλεια Λιόντη'), - ('=?utf-8?b?15nXqdeo15DXnCDXqNeV15bXoNek15zXkw==?=', 'ישראל רוזנפלד'), - ('=?utf-8?b?5Li95Y2OIOeahw==?=', '丽华 皇'), - ('=?utf-8?b?77ul77qu766V77qzIO+tlu+7ru+vvu+6ju+7pw==?=', 'ﻥﺮﮕﺳ ﭖﻮﯾﺎﻧ'), - ('=?utf-8?b?77uh77uu77qz77uu76++IO+6su+7tO+7p++6jSDvurDvu6Pvuo7vu6jvr74=?=', 'ﻡﻮﺳﻮﯾ ﺲﻴﻧﺍ ﺰﻣﺎﻨﯾ'), - ('=?utf-8?b?ScOxaWdvIFNhbsOnIEliw6HDsWV6IGRlIGxhIFBlw7Fh?=', 'Iñigo Sanç Ibáñez de la Peña'), - ('Mart van Oostendorp', 'Mart van Oostendorp'), - ('', ''), - ) - for encoded_str, unicode in names: - self.assertEqual(unicode, parse_unicode(encoded_str)) - class TestAndroidSiteManifest(TestCase): def test_manifest(self): r = self.client.get(urlreverse('site.webmanifest')) diff --git a/ietf/utils/tests_coverage.py b/ietf/utils/tests_coverage.py new file mode 100644 index 0000000000..68795994a7 --- /dev/null +++ b/ietf/utils/tests_coverage.py @@ -0,0 +1,56 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +"""Tests of the coverage.py module""" + +from unittest import mock + +from django.test import override_settings + +from .coverage import CoverageManager +from .test_utils import TestCase + + +class CoverageManagerTests(TestCase): + @override_settings( + BASE_DIR="/path/to/project/ietf", + TEST_CODE_COVERAGE_EXCLUDE_FILES=["a.py"], + TEST_CODE_COVERAGE_EXCLUDE_LINES=["some-regex"], + ) + @mock.patch("ietf.utils.coverage.Coverage") + def test_coverage_manager(self, mock_coverage): + """CoverageManager managed coverage correctly in non-production mode + + Presumes we're not running tests in production mode. + """ + cm = CoverageManager() + self.assertFalse(cm.started) + + cm.start() + self.assertTrue(cm.started) + self.assertEqual(cm.checker, mock_coverage.return_value) + self.assertTrue(mock_coverage.called) + coverage_kwargs = mock_coverage.call_args.kwargs + self.assertEqual(coverage_kwargs["source"], ["/path/to/project/ietf"]) + self.assertEqual(coverage_kwargs["omit"], ["a.py"]) + self.assertTrue(isinstance(cm.checker.exclude, mock.Mock)) + assert isinstance(cm.checker.exclude, mock.Mock) # for type checker + self.assertEqual(cm.checker.exclude.call_count, 1) + cm.checker.exclude.assert_called_with("some-regex") + + @mock.patch("ietf.utils.coverage.Coverage") + def test_coverage_manager_is_defanged_in_production(self, mock_coverage): + """CoverageManager is a no-op in production mode""" + # Be careful faking settings.SERVER_MODE, but there's really no other way to + # test this. + with override_settings(SERVER_MODE="production"): + cm = CoverageManager() + cm.start() + + # Check that nothing actually happened + self.assertFalse(mock_coverage.called) + self.assertIsNone(cm.checker) + self.assertFalse(cm.started) + + # Check that other methods are guarded appropriately + cm.stop() + cm.save() + self.assertIsNone(cm.report()) diff --git a/ietf/utils/tests_meetecho.py b/ietf/utils/tests_meetecho.py index a10ac68c27..c076a3df74 100644 --- a/ietf/utils/tests_meetecho.py +++ b/ietf/utils/tests_meetecho.py @@ -98,7 +98,7 @@ def test_schedule_meeting(self): api_response = api.schedule_meeting( wg_token='my-token', room_id=18, - start_time=datetime.datetime(2021, 9, 14, 10, 0, 0, tzinfo=datetime.timezone.utc), + start_time=datetime.datetime(2021, 9, 14, 10, 0, 0, tzinfo=datetime.UTC), duration=datetime.timedelta(minutes=130), description='interim-2021-wgname-01', extrainfo='message for staff', @@ -127,7 +127,7 @@ def test_schedule_meeting(self): ) # same time in different time zones for start_time in [ - datetime.datetime(2021, 9, 14, 10, 0, 0, tzinfo=datetime.timezone.utc), + datetime.datetime(2021, 9, 14, 10, 0, 0, tzinfo=datetime.UTC), datetime.datetime(2021, 9, 14, 7, 0, 0, tzinfo=ZoneInfo('America/Halifax')), datetime.datetime(2021, 9, 14, 13, 0, 0, tzinfo=ZoneInfo('Europe/Kiev')), datetime.datetime(2021, 9, 14, 5, 0, 0, tzinfo=ZoneInfo('Pacific/Easter')), @@ -198,7 +198,7 @@ def test_fetch_meetings(self): '3d55bce0-535e-4ba8-bb8e-734911cf3c32': { 'room': { 'id': 18, - 'start_time': datetime.datetime(2021, 9, 14, 10, 0, 0, tzinfo=datetime.timezone.utc), + 'start_time': datetime.datetime(2021, 9, 14, 10, 0, 0, tzinfo=datetime.UTC), 'duration': datetime.timedelta(minutes=130), 'description': 'interim-2021-wgname-01', }, @@ -208,7 +208,7 @@ def test_fetch_meetings(self): 'e68e96d4-d38f-475b-9073-ecab46ca96a5': { 'room': { 'id': 23, - 'start_time': datetime.datetime(2021, 9, 15, 14, 30, 0, tzinfo=datetime.timezone.utc), + 'start_time': datetime.datetime(2021, 9, 15, 14, 30, 0, tzinfo=datetime.UTC), 'duration': datetime.timedelta(minutes=30), 'description': 'interim-2021-wgname-02', }, @@ -386,7 +386,7 @@ def test_request_helper_exception(self): def test_time_serialization(self): """Time de/serialization should be consistent""" - time = timezone.now().astimezone(datetime.timezone.utc).replace(microsecond=0) # cut off to 0 microseconds + time = timezone.now().astimezone(datetime.UTC).replace(microsecond=0) # cut off to 0 microseconds api = MeetechoAPI(API_BASE, CLIENT_ID, CLIENT_SECRET) self.assertEqual(api._deserialize_time(api._serialize_time(time)), time) @@ -400,7 +400,7 @@ def test_conference_from_api_dict(self): 'session-1-uuid': { 'room': { 'id': 1, - 'start_time': datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.timezone.utc), + 'start_time': datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.UTC), 'duration': datetime.timedelta(minutes=45), 'description': 'some-description', }, @@ -410,7 +410,7 @@ def test_conference_from_api_dict(self): 'session-2-uuid': { 'room': { 'id': 2, - 'start_time': datetime.datetime(2022,2,5,4,5,6, tzinfo=datetime.timezone.utc), + 'start_time': datetime.datetime(2022,2,5,4,5,6, tzinfo=datetime.UTC), 'duration': datetime.timedelta(minutes=90), 'description': 'another-description', }, @@ -427,7 +427,7 @@ def test_conference_from_api_dict(self): id=1, public_id='session-1-uuid', description='some-description', - start_time=datetime.datetime(2022, 2, 4, 1, 2, 3, tzinfo=datetime.timezone.utc), + start_time=datetime.datetime(2022, 2, 4, 1, 2, 3, tzinfo=datetime.UTC), duration=datetime.timedelta(minutes=45), url='https://example.com/some/url', deletion_token='delete-me', @@ -437,7 +437,7 @@ def test_conference_from_api_dict(self): id=2, public_id='session-2-uuid', description='another-description', - start_time=datetime.datetime(2022, 2, 5, 4, 5, 6, tzinfo=datetime.timezone.utc), + start_time=datetime.datetime(2022, 2, 5, 4, 5, 6, tzinfo=datetime.UTC), duration=datetime.timedelta(minutes=90), url='https://example.com/another/url', deletion_token='delete-me-too', @@ -453,7 +453,7 @@ def test_fetch(self, mock_fetch, _): 'session-1-uuid': { 'room': { 'id': 1, - 'start_time': datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.timezone.utc), + 'start_time': datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.UTC), 'duration': datetime.timedelta(minutes=45), 'description': 'some-description', }, @@ -472,7 +472,7 @@ def test_fetch(self, mock_fetch, _): id=1, public_id='session-1-uuid', description='some-description', - start_time=datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.timezone.utc), + start_time=datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.UTC), duration=datetime.timedelta(minutes=45), url='https://example.com/some/url', deletion_token='delete-me', @@ -488,7 +488,7 @@ def test_create(self, mock_schedule, _): 'session-1-uuid': { 'room': { 'id': 1, # value should match session_id param to cm.create() below - 'start_time': datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.timezone.utc), + 'start_time': datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.UTC), 'duration': datetime.timedelta(minutes=45), 'description': 'some-description', }, @@ -506,7 +506,7 @@ def test_create(self, mock_schedule, _): id=1, public_id='session-1-uuid', description='some-description', - start_time=datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.timezone.utc), + start_time=datetime.datetime(2022,2,4,1,2,3, tzinfo=datetime.UTC), duration=datetime.timedelta(minutes=45), url='https://example.com/some/url', deletion_token='delete-me', @@ -547,7 +547,8 @@ def test_add(self, mock_add, mock_wg_token): sm = SlidesManager(settings.MEETECHO_API_CONFIG) session = SessionFactory() slides_doc = DocumentFactory(type_id="slides") - sm.add(session, slides_doc, 13) + retval = sm.add(session, slides_doc, 13) + self.assertIs(retval, True) self.assertTrue(mock_wg_token.called) self.assertTrue(mock_add.called) self.assertEqual( @@ -565,6 +566,14 @@ def test_add(self, mock_add, mock_wg_token): ), ) + # Test return value when no update is sent. Really ought to do a more + # careful test of the _should_send_update() method. + sm = SlidesManager( + settings.MEETECHO_API_CONFIG | {"slides_notify_time": None} + ) + retval = sm.add(session, slides_doc, 14) + self.assertIs(retval, False) + @patch("ietf.utils.meetecho.MeetechoAPI.update_slide_decks") @patch("ietf.utils.meetecho.MeetechoAPI.delete_slide_deck") def test_delete(self, mock_delete, mock_update, mock_wg_token): @@ -580,7 +589,8 @@ def test_delete(self, mock_delete, mock_update, mock_wg_token): sm.delete(session, slides_doc) # can't remove slides still attached to the session self.assertFalse(any([mock_wg_token.called, mock_delete.called, mock_update.called])) - sm.delete(session, removed_slides_doc) + retval = sm.delete(session, removed_slides_doc) + self.assertIs(retval, True) self.assertTrue(mock_wg_token.called) self.assertTrue(mock_delete.called) self.assertEqual( @@ -609,9 +619,18 @@ def test_delete(self, mock_delete, mock_update, mock_wg_token): # Delete the other session and check that we don't make the update call slides.delete() - sm.delete(session, slides_doc) + retval = sm.delete(session, slides_doc) + self.assertIs(retval, True) self.assertTrue(mock_delete.called) self.assertFalse(mock_update.called) + + # Test return value when no update is sent. Really ought to do a more + # careful test of the _should_send_update() method. + sm = SlidesManager( + settings.MEETECHO_API_CONFIG | {"slides_notify_time": None} + ) + retval = sm.delete(session, slides_doc) + self.assertIs(retval, False) @patch("ietf.utils.meetecho.MeetechoAPI.delete_slide_deck") @patch("ietf.utils.meetecho.MeetechoAPI.add_slide_deck") @@ -619,7 +638,8 @@ def test_revise(self, mock_add, mock_delete, mock_wg_token): sm = SlidesManager(settings.MEETECHO_API_CONFIG) slides = SessionPresentationFactory(document__type_id="slides", order=23) slides_doc = slides.document - sm.revise(slides.session, slides.document) + retval = sm.revise(slides.session, slides_doc) + self.assertIs(retval, True) self.assertTrue(mock_wg_token.called) self.assertTrue(mock_delete.called) self.assertEqual( @@ -642,13 +662,22 @@ def test_revise(self, mock_add, mock_delete, mock_wg_token): ), ) + # Test return value when no update is sent. Really ought to do a more + # careful test of the _should_send_update() method. + sm = SlidesManager( + settings.MEETECHO_API_CONFIG | {"slides_notify_time": None} + ) + retval = sm.revise(slides.session, slides_doc) + self.assertIs(retval, False) + @patch("ietf.utils.meetecho.MeetechoAPI.update_slide_decks") def test_send_update(self, mock_send_update, mock_wg_token): sm = SlidesManager(settings.MEETECHO_API_CONFIG) slides = SessionPresentationFactory(document__type_id="slides") SessionPresentationFactory(session=slides.session, document__type_id="agenda") - sm.send_update(slides.session) + retval = sm.send_update(slides.session) + self.assertIs(retval, True) self.assertTrue(mock_wg_token.called) self.assertTrue(mock_send_update.called) self.assertEqual( @@ -667,3 +696,11 @@ def test_send_update(self, mock_send_update, mock_wg_token): ] ) ) + + # Test return value when no update is sent. Really ought to do a more + # careful test of the _should_send_update() method. + sm = SlidesManager( + settings.MEETECHO_API_CONFIG | {"slides_notify_time": None} + ) + retval = sm.send_update(slides.session) + self.assertIs(retval, False) diff --git a/ietf/utils/tests_searchindex.py b/ietf/utils/tests_searchindex.py new file mode 100644 index 0000000000..b97bc2e266 --- /dev/null +++ b/ietf/utils/tests_searchindex.py @@ -0,0 +1,325 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +from unittest import mock + +import requests.exceptions +import typesense.exceptions +from django.conf import settings +from django.test.utils import override_settings + +from . import searchindex +from .test_utils import TestCase +from ..blobdb.models import Blob +from ..doc.factories import ( + WgDraftFactory, + WgRfcFactory, + PublishedRfcDocEventFactory, + BcpFactory, + StdFactory, +) +from ..doc.models import Document, RelatedDocument +from ..doc.storage_utils import store_str +from ..person.factories import PersonFactory + + +class SearchindexTests(TestCase): + def test_enabled(self): + with override_settings(): + try: + del settings.SEARCHINDEX_CONFIG + except AttributeError: + pass + self.assertFalse(searchindex.enabled()) + with override_settings( + SEARCHINDEX_CONFIG={"TYPESENSE_API_KEY": "this-is-not-a-key"} + ): + self.assertFalse(searchindex.enabled()) + with override_settings( + SEARCHINDEX_CONFIG={"TYPESENSE_API_URL": "http://example.com"} + ): + self.assertTrue(searchindex.enabled()) + + def test_sanitize_text(self): + dirty_text = """ + + This is text. It + is <---- full of \tprobl.....ems! Fix it. + """ + sanitized = "This is text It is full of problems Fix it." + self.assertEqual(searchindex._sanitize_text(dirty_text), sanitized) + + def test_sanitize_abstract(self): + dirty_abstract = ( + "Mixed\n" + "Newlines\r" + "And\r\n" + "Things\n\r" + " Sometimes\n" + "\n" + "With\r\n" + "\r\n" + "Double \n\r" + "\n\r" + " Newlines\r" + "\r" + "Whee!" + ) + sanitized = ( + "Mixed\n" + "Newlines\n" + "And\n" + "Things\n" + "Sometimes\n" + "\n" + "With\n" + "\n" + "Double\n" + "\n" + "Newlines\n" + "\n" + "Whee!" + ) + self.assertEqual(searchindex._sanitize_abstract(dirty_abstract), sanitized) + + def test_typesense_doc_from_rfc(self): + not_rfc = WgDraftFactory() + assert isinstance(not_rfc, Document) + with self.assertRaises(AssertionError): + searchindex.typesense_doc_from_rfc(not_rfc) + + invalid_rfc = WgRfcFactory(name="rfc1000000", rfc_number=None) + assert isinstance(invalid_rfc, Document) + with self.assertRaises(AssertionError): + searchindex.typesense_doc_from_rfc(invalid_rfc) + + rfc = PublishedRfcDocEventFactory().doc + assert isinstance(rfc, Document) + result = searchindex.typesense_doc_from_rfc(rfc) + # Check a few values, not exhaustive + self.assertEqual(result["id"], f"doc-{rfc.pk}") + self.assertEqual(result["rfcNumber"], rfc.rfc_number) + self.assertEqual(result["abstract"], searchindex._sanitize_abstract(rfc.abstract)) + self.assertEqual(result["pages"], rfc.pages) + self.assertNotIn("adName", result) + self.assertNotIn("content", result) # no blob + self.assertNotIn("subseries", result) + + # repeat, this time with contents, an AD, and subseries docs + store_str( + kind="rfc", + name=f"txt/{rfc.name}.txt", + content="The contents of this RFC", + doc_name=rfc.name, + doc_rev=rfc.rev, # expected to be None + ) + rfc.ad = PersonFactory(name="Alfred D. Rector") + # Put it in two Subseries docs to be sure this does not break things + # (the typesense schema does not support this for real at the moment) + BcpFactory(contains=[rfc], name="bcp1234") + StdFactory(contains=[rfc], name="std1234") + result = searchindex.typesense_doc_from_rfc(rfc) + # Check a few values, not exhaustive + self.assertEqual( + result["content"], + searchindex._sanitize_text("The contents of this RFC"), + ) + self.assertEqual(result["adName"], "Alfred D. Rector") + self.assertIn("subseries", result) + ss_dict = result["subseries"] + # We should get one of the two subseries docs, but neither is more correct + # than the other... + self.assertTrue( + any( + ss_dict == {"acronym": ss_type, "number": 1234, "total": 1} + for ss_type in ["bcp", "std"] + ) + ) + + # Finally, delete the contents blob and make sure things don't blow up + Blob.objects.get(bucket="rfc", name=f"txt/{rfc.name}.txt").delete() + result = searchindex.typesense_doc_from_rfc(rfc) + self.assertNotIn("content", result) + + def test_typesense_doc_from_rfc_flags_obsoleted(self): + """typesense docs should set correct flags for obsoleted RFC""" + rfc = PublishedRfcDocEventFactory().doc + assert isinstance(rfc, Document) + self.assertEqual(len(rfc.related_that("obs")), 0) + self.assertEqual(len(rfc.related_that("updates")), 0) + self.assertNotEqual(rfc.std_level.slug, "hist") + result = searchindex.typesense_doc_from_rfc(rfc) + self.assertFalse(result["flags"]["hiddenDefault"]) + self.assertFalse(result["flags"]["obsoleted"]) + self.assertFalse(result["flags"]["updated"]) + + RelatedDocument.objects.create( + source=(PublishedRfcDocEventFactory().doc), + target=rfc, + relationship_id="obs", + ) + result = searchindex.typesense_doc_from_rfc(rfc) + self.assertTrue(result["flags"]["hiddenDefault"]) + self.assertTrue(result["flags"]["obsoleted"]) + self.assertFalse(result["flags"]["updated"]) + + def test_typesense_doc_from_rfc_flags_updated(self): + """typesense docs should set flags correctly for updated RFC""" + rfc = PublishedRfcDocEventFactory().doc + assert isinstance(rfc, Document) + self.assertEqual(len(rfc.related_that("obs")), 0) + self.assertEqual(len(rfc.related_that("updates")), 0) + self.assertNotEqual(rfc.std_level.slug, "hist") + result = searchindex.typesense_doc_from_rfc(rfc) + self.assertFalse(result["flags"]["hiddenDefault"]) + self.assertFalse(result["flags"]["obsoleted"]) + self.assertFalse(result["flags"]["updated"]) + + RelatedDocument.objects.create( + source=(PublishedRfcDocEventFactory().doc), + target=rfc, + relationship_id="updates", + ) + result = searchindex.typesense_doc_from_rfc(rfc) + self.assertFalse(result["flags"]["hiddenDefault"]) + self.assertFalse(result["flags"]["obsoleted"]) + self.assertTrue(result["flags"]["updated"]) + + def test_typesense_doc_from_rfc_flags_historic(self): + """typesense docs should set flags correctly for historic RFC""" + rfc = PublishedRfcDocEventFactory(doc__std_level_id="hist").doc + assert isinstance(rfc, Document) + result = searchindex.typesense_doc_from_rfc(rfc) + self.assertTrue(result["flags"]["hiddenDefault"]) + self.assertFalse(result["flags"]["obsoleted"]) + self.assertFalse(result["flags"]["updated"]) + + + @override_settings( + SEARCHINDEX_CONFIG={ + "TYPESENSE_API_URL": "http://ts.example.com", + "TYPESENSE_API_KEY": "test-api-key", + "TYPESENSE_COLLECTION_NAME": "frogs", + } + ) + @mock.patch("ietf.utils.searchindex.typesense_doc_from_rfc") + @mock.patch("ietf.utils.searchindex.typesense.Client") + def test_update_or_create_rfc_entry( + self, mock_ts_client_constructor, mock_tdoc_from_rfc + ): + fake_tdoc = object() + mock_tdoc_from_rfc.return_value = fake_tdoc + rfc = WgRfcFactory() + assert isinstance(rfc, Document) + searchindex.update_or_create_rfc_entry(rfc) + self.assertTrue(mock_ts_client_constructor.called) + # walk the tree down to the method we expected to be called... + mock_upsert = mock_ts_client_constructor.return_value.collections[ + "frogs" # matches value in override_settings above + ].documents.upsert + self.assertTrue(mock_upsert.called) + self.assertEqual(mock_upsert.call_args, mock.call(fake_tdoc)) + + @override_settings( + SEARCHINDEX_CONFIG={ + "TYPESENSE_API_URL": "http://ts.example.com", + "TYPESENSE_API_KEY": "test-api-key", + "TYPESENSE_COLLECTION_NAME": "frogs", + } + ) + @mock.patch("ietf.utils.searchindex.typesense_doc_from_rfc") + @mock.patch("ietf.utils.searchindex.typesense.Client") + def test_update_or_create_rfc_entries( + self, mock_ts_client_constructor, mock_tdoc_from_rfc + ): + fake_tdoc = object() + mock_tdoc_from_rfc.return_value = fake_tdoc + rfc = WgRfcFactory() + assert isinstance(rfc, Document) + searchindex.update_or_create_rfc_entries([rfc] * 50) # list of docs... + self.assertEqual(mock_ts_client_constructor.call_count, 1) + # walk the tree down to the method we expected to be called... + mock_import_ = mock_ts_client_constructor.return_value.collections[ + "frogs" # matches value in override_settings above + ].documents.import_ + self.assertEqual(mock_import_.call_count, 1) + self.assertEqual( + mock_import_.call_args, mock.call([fake_tdoc] * 50, {"action": "upsert"}) + ) + + mock_import_.reset_mock() + searchindex.update_or_create_rfc_entries([rfc] * 50, batchsize=20) + self.assertEqual(mock_ts_client_constructor.call_count, 2) # one more + # walk the tree down to the method we expected to be called... + mock_import_ = mock_ts_client_constructor.return_value.collections[ + "frogs" # matches value in override_settings above + ].documents.import_ + self.assertEqual(mock_import_.call_count, 3) + self.assertEqual( + mock_import_.call_args_list, + [ + mock.call([fake_tdoc] * 20, {"action": "upsert"}), + mock.call([fake_tdoc] * 20, {"action": "upsert"}), + mock.call([fake_tdoc] * 10, {"action": "upsert"}), + ], + ) + + @override_settings( + SEARCHINDEX_CONFIG={ + "TYPESENSE_API_URL": "http://ts.example.com", + "TYPESENSE_API_KEY": "test-api-key", + "TYPESENSE_COLLECTION_NAME": "frogs", + } + ) + @mock.patch("ietf.utils.searchindex.typesense.Client") + def test_create_collection(self, mock_ts_client_constructor): + searchindex.create_collection() + self.assertEqual(mock_ts_client_constructor.call_count, 1) + mock_collections = mock_ts_client_constructor.return_value.collections + self.assertTrue(mock_collections.create.called) + self.assertEqual(mock_collections.create.call_args[0][0]["name"], "frogs") + + @override_settings( + SEARCHINDEX_CONFIG={ + "TYPESENSE_API_URL": "http://ts.example.com", + "TYPESENSE_API_KEY": "test-api-key", + "TYPESENSE_COLLECTION_NAME": "frogs", + } + ) + @mock.patch("ietf.utils.searchindex.typesense.Client") + def test_delete_collection(self, mock_ts_client_constructor): + searchindex.delete_collection() + self.assertEqual(mock_ts_client_constructor.call_count, 1) + mock_collections = mock_ts_client_constructor.return_value.collections + self.assertTrue(mock_collections["frogs"].delete.called) + + mock_collections["frogs"].side_effect = typesense.exceptions.ObjectNotFound + searchindex.delete_collection() # should ignore the exception + + @override_settings( + SEARCHINDEX_CONFIG={ + "TYPESENSE_API_URL": "http://ts.example.com", + "TYPESENSE_API_KEY": "test-api-key", + "TYPESENSE_COLLECTION_NAME": "frogs", + } + ) + def test_upsert_presets(self): + self.requests_mock.put( + "http://ts.example.com/presets/red", text="ok", status_code=201 + ) + self.requests_mock.put( + "http://ts.example.com/presets/red-content", text="ok", status_code=202 + ) + searchindex.upsert_presets() + + self.requests_mock.put( + "http://ts.example.com/presets/red", text="not ok", status_code=400 + ) + with self.assertRaises(requests.exceptions.HTTPError): + searchindex.upsert_presets() + + self.requests_mock.put( + "http://ts.example.com/presets/red", text="ok", status_code=200 + ) + self.requests_mock.put( + "http://ts.example.com/presets/red-content", text="not ok", status_code=400 + ) + with self.assertRaises(requests.exceptions.HTTPError): + searchindex.upsert_presets() diff --git a/ietf/utils/tests_text.py b/ietf/utils/tests_text.py new file mode 100644 index 0000000000..51aa2eff13 --- /dev/null +++ b/ietf/utils/tests_text.py @@ -0,0 +1,71 @@ +# Copyright The IETF Trust 2021-2026, All Rights Reserved +from ietf.utils.test_utils import TestCase +from ietf.utils.text import parse_unicode, decode_document_content + + +class TestDecoders(TestCase): + def test_parse_unicode(self): + names = ( + ("=?utf-8?b?4Yuz4YuK4Ym1IOGJoOGJgOGIiA==?=", "ዳዊት በቀለ"), + ("=?utf-8?b?5Li9IOmDnA==?=", "丽 郜"), + ("=?utf-8?b?4KSV4KSu4KWN4KSs4KWL4KScIOCkoeCkvuCksA==?=", "कम्बोज डार"), + ("=?utf-8?b?zpfPgc6szrrOu861zrnOsSDOm865z4zOvc+Ezrc=?=", "Ηράκλεια Λιόντη"), + ("=?utf-8?b?15nXqdeo15DXnCDXqNeV15bXoNek15zXkw==?=", "ישראל רוזנפלד"), + ("=?utf-8?b?5Li95Y2OIOeahw==?=", "丽华 皇"), + ("=?utf-8?b?77ul77qu766V77qzIO+tlu+7ru+vvu+6ju+7pw==?=", "ﻥﺮﮕﺳ ﭖﻮﯾﺎﻧ"), + ( + "=?utf-8?b?77uh77uu77qz77uu76++IO+6su+7tO+7p++6jSDvurDvu6Pvuo7vu6jvr74=?=", + "ﻡﻮﺳﻮﯾ ﺲﻴﻧﺍ ﺰﻣﺎﻨﯾ", + ), + ( + "=?utf-8?b?ScOxaWdvIFNhbsOnIEliw6HDsWV6IGRlIGxhIFBlw7Fh?=", + "Iñigo Sanç Ibáñez de la Peña", + ), + ("Mart van Oostendorp", "Mart van Oostendorp"), + ("", ""), + ) + for encoded_str, unicode in names: + self.assertEqual(unicode, parse_unicode(encoded_str)) + + def test_decode_document_content(self): + utf8_bytes = "𒀭𒊩𒌆𒄈𒋢".encode("utf-8") # ends with 4-byte character + latin1_bytes = "àéîøü".encode("latin-1") + other_bytes = "àéîøü".encode("macintosh") # different from its latin-1 encoding + assert other_bytes.decode("macintosh") != other_bytes.decode("latin-1"),\ + "test broken: other_bytes must decode differently as latin-1" + + # simplest case + self.assertEqual( + decode_document_content(utf8_bytes), + utf8_bytes.decode(), + ) + # losing 1-4 bytes from the end leave the last character incomplete; the + # decoder should decode all but that last character + self.assertEqual( + decode_document_content(utf8_bytes[:-1]), + utf8_bytes.decode()[:-1], + ) + self.assertEqual( + decode_document_content(utf8_bytes[:-2]), + utf8_bytes.decode()[:-1], + ) + self.assertEqual( + decode_document_content(utf8_bytes[:-3]), + utf8_bytes.decode()[:-1], + ) + self.assertEqual( + decode_document_content(utf8_bytes[:-4]), + utf8_bytes.decode()[:-1], + ) + + # latin-1 is also simple + self.assertEqual( + decode_document_content(latin1_bytes), + latin1_bytes.decode("latin-1"), + ) + + # other character sets are just treated as latin1 (bug? feature? you decide) + self.assertEqual( + decode_document_content(other_bytes), + other_bytes.decode("latin-1"), + ) diff --git a/ietf/utils/text.py b/ietf/utils/text.py index 4e5d5b6cd5..2763056e1a 100644 --- a/ietf/utils/text.py +++ b/ietf/utils/text.py @@ -60,6 +60,12 @@ def check_url_validity(attrs, new=False): def linkify(text): + """Convert URL-ish substrings into HTML links + + This does no sanitization whatsoever. Caller must sanitize the input or output as + contextually appropriate. Do not call `mark_safe()` on the output if the input is + user-provided unless it has been sanitized or escaped. + """ return _bleach_linker.linkify(text) @@ -257,3 +263,21 @@ def parse_unicode(text): else: text = decoded_string return text + + +def decode_document_content(content: bytes) -> str: + """Decode document contents as utf-8 or latin1 + + Method was developed in DocumentInfo.text() where it gave acceptable results + for existing documents / RFCs. + """ + try: + return content.decode("utf-8") + except UnicodeDecodeError: + pass + for back in range(1, 4): + try: + return content[:-back].decode("utf-8") + except UnicodeDecodeError: + pass + return content.decode("latin-1") # everything is legal in latin-1 diff --git a/ietf/utils/timezone.py b/ietf/utils/timezone.py index a396b5e82d..e08dfa02f2 100644 --- a/ietf/utils/timezone.py +++ b/ietf/utils/timezone.py @@ -26,7 +26,7 @@ def _tzinfo(tz: Union[str, datetime.tzinfo, None]): Accepts a tzinfo or string containing a timezone name. Defaults to UTC if tz is None. """ if tz is None: - return datetime.timezone.utc + return datetime.UTC elif isinstance(tz, datetime.tzinfo): return tz else: diff --git a/ietf/utils/unicodenormalize.py b/ietf/utils/unicodenormalize.py new file mode 100644 index 0000000000..8644dbdb79 --- /dev/null +++ b/ietf/utils/unicodenormalize.py @@ -0,0 +1,9 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +import unicodedata + +def normalize_for_sorting(text): + """Normalize text for proper accent-aware sorting.""" + # Normalize the text to NFD (decomposed form) + decomposed = unicodedata.normalize('NFD', text) + # Filter out combining diacritical marks + return ''.join(char for char in decomposed if not unicodedata.combining(char)) diff --git a/ietf/utils/validators.py b/ietf/utils/validators.py index 8fe989df99..a99de72724 100644 --- a/ietf/utils/validators.py +++ b/ietf/utils/validators.py @@ -4,6 +4,8 @@ import os import re +from email.utils import parseaddr + from pyquery import PyQuery from urllib.parse import urlparse, urlsplit, urlunsplit @@ -11,7 +13,13 @@ from django.apps import apps from django.conf import settings from django.core.exceptions import ObjectDoesNotExist, ValidationError -from django.core.validators import RegexValidator, URLValidator, EmailValidator, BaseValidator +from django.core.validators import ( + RegexValidator, + URLValidator, + BaseValidator, + validate_email, + ProhibitNullCharactersValidator, +) from django.template.defaultfilters import filesizeformat from django.utils.deconstruct import deconstructible from django.utils.ipv6 import is_valid_ipv6_address @@ -25,8 +33,9 @@ # Note that this is an instantiation of the regex validator, _not_ the # regex-string validator defined right below validate_no_control_chars = RegexValidator( - regex="^[^\x00-\x1f]*$", - message="Please enter a string without control characters." ) + regex="^[^\x01-\x1f]*$", + message="Please enter a string without control characters.", +) @deconstructible @@ -136,8 +145,17 @@ def validate_no_html_frame(file): # instantiations of sub-validiators used by the external_resource validator validate_url = URLValidator() -validate_http_url = URLValidator(schemes=['http','https']) -validate_email = EmailValidator() +validate_http_url = URLValidator(schemes=["http", "https"]) +validate_no_nulls = ProhibitNullCharactersValidator() + + +def validate_mailbox_address(s): + """Validate an RFC 5322 'mailbox' (e.g., "Some Person" )""" + # parseaddr() returns ("", "") on err; validate_email() will reject that for us + name, addr = parseaddr(s) + validate_no_nulls(name) # could be stricter... + validate_email(addr) + def validate_ipv6_address(value): if not is_valid_ipv6_address(value): diff --git a/ietf/utils/xmldraft.py b/ietf/utils/xmldraft.py index 3ac9a269c7..325b8499a9 100644 --- a/ietf/utils/xmldraft.py +++ b/ietf/utils/xmldraft.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2022, All Rights Reserved +# Copyright The IETF Trust 2022-2025, All Rights Reserved # -*- coding: utf-8 -*- import datetime import io @@ -102,6 +102,17 @@ def _document_name(self, ref): number = int(maybe_number) return f"{label}{number}" + target = ref.get("target") + if isinstance(target, str): + target = target.lower() + if target.startswith("https://datatracker.ietf.org/doc/"): + # len("https://datatracker.ietf.org/doc/")==33 + m = re.match(r"^(draft-[a-z0-9-]*[a-z0-9])([/-]\d{2})?/?$",target[33:]) + if m: + name = m.group(1) + return name + + # if we couldn't find a match so far, try the seriesInfo series_query = " or ".join(f"@name='{x.upper()}'" for x in series) for info in ref.xpath( @@ -147,7 +158,8 @@ def parse_docname(xmlroot): return revmatch.group('filename'), revmatch.group('rev') def get_title(self): - return self.xmlroot.findtext('front/title').strip() + title_text = self.xmlroot.findtext('front/title') + return "" if title_text is None else title_text.strip() @staticmethod def parse_creation_date(date_elt): @@ -232,6 +244,12 @@ def render_author_name(author_elt): # Use fullname attribute, if present fullname = author_elt.attrib.get("fullname", "").strip() if fullname: + # If any 8bit chars in the fullname, try to append the author's + # name in ascii. + if any([x >= 0x80 for x in fullname.encode('utf8')]): + asciifullname = author_elt.attrib.get("asciiFullname", "").strip() + if asciifullname: + fullname = fullname + ' (' + asciifullname + ')' return fullname surname = author_elt.attrib.get("surname", "").strip() initials = author_elt.attrib.get("initials", "").strip() diff --git a/k8s/auth.yaml b/k8s/auth.yaml index 392e306b54..992c90557a 100644 --- a/k8s/auth.yaml +++ b/k8s/auth.yaml @@ -9,22 +9,12 @@ spec: matchLabels: app: auth strategy: - type: Recreate + type: DEPLOY_STRATEGY template: metadata: labels: app: auth spec: - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - datatracker - topologyKey: "kubernetes.io/hostname" securityContext: runAsNonRoot: true containers: @@ -71,6 +61,10 @@ spec: readOnlyRootFilesystem: true runAsUser: 1000 runAsGroup: 1000 + resources: + requests: + cpu: 250m + memory: 4Gi # ----------------------------------------------------- # Nginx Container # ----------------------------------------------------- @@ -96,6 +90,10 @@ spec: - name: dt-cfg mountPath: /etc/nginx/conf.d/default.conf subPath: nginx-auth.conf + resources: + requests: + cpu: 10m + memory: 10Mi # ----------------------------------------------------- # ScoutAPM Container # ----------------------------------------------------- @@ -121,6 +119,10 @@ spec: readOnlyRootFilesystem: true runAsUser: 65534 # "nobody" user by default runAsGroup: 65534 # "nogroup" group by default + resources: + requests: + cpu: 10m + memory: 50Mi volumes: # To be overriden with the actual shared volume - name: dt-vol @@ -139,8 +141,6 @@ spec: - name: nginx-tmp emptyDir: sizeLimit: "500Mi" - dnsPolicy: ClusterFirst - restartPolicy: Always terminationGracePeriodSeconds: 60 --- apiVersion: v1 diff --git a/k8s/beat.yaml b/k8s/beat.yaml index 9a8fe2f0a4..cc171fb7d1 100644 --- a/k8s/beat.yaml +++ b/k8s/beat.yaml @@ -17,16 +17,6 @@ spec: labels: app: beat spec: - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - datatracker - topologyKey: "kubernetes.io/hostname" securityContext: runAsNonRoot: true containers: @@ -36,10 +26,6 @@ spec: - name: beat image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG" imagePullPolicy: Always - ports: - - containerPort: 8000 - name: http - protocol: TCP volumeMounts: - name: dt-vol mountPath: /a @@ -62,8 +48,12 @@ spec: readOnlyRootFilesystem: true runAsUser: 1000 runAsGroup: 1000 + resources: + requests: + cpu: 100m + memory: 250Mi volumes: - # To be overriden with the actual shared volume + # To be overridden with the actual shared volume - name: dt-vol - name: dt-tmp emptyDir: @@ -73,4 +63,4 @@ spec: name: files-cfgmap dnsPolicy: ClusterFirst restartPolicy: Always - terminationGracePeriodSeconds: 600 + terminationGracePeriodSeconds: 10 diff --git a/k8s/celery.yaml b/k8s/celery.yaml index 15f3bf0c7c..f6cea2acc7 100644 --- a/k8s/celery.yaml +++ b/k8s/celery.yaml @@ -17,16 +17,6 @@ spec: labels: app: celery spec: - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - datatracker - topologyKey: "kubernetes.io/hostname" securityContext: runAsNonRoot: true containers: @@ -36,10 +26,6 @@ spec: - name: celery image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG" imagePullPolicy: Always - ports: - - containerPort: 8000 - name: http - protocol: TCP volumeMounts: - name: dt-vol mountPath: /a @@ -66,6 +52,10 @@ spec: readOnlyRootFilesystem: true runAsUser: 1000 runAsGroup: 1000 + resources: + requests: + cpu: 100m + memory: 1Gi # ----------------------------------------------------- # ScoutAPM Container # ----------------------------------------------------- @@ -91,8 +81,12 @@ spec: readOnlyRootFilesystem: true runAsUser: 65534 # "nobody" user by default runAsGroup: 65534 # "nogroup" group by default + resources: + requests: + cpu: 10m + memory: 50Mi volumes: - # To be overriden with the actual shared volume + # To be overridden with the actual shared volume - name: dt-vol - name: dt-tmp emptyDir: diff --git a/k8s/datatracker.yaml b/k8s/datatracker.yaml index 3d9e86a29d..ff89fb3722 100644 --- a/k8s/datatracker.yaml +++ b/k8s/datatracker.yaml @@ -9,7 +9,7 @@ spec: matchLabels: app: datatracker strategy: - type: Recreate + type: DEPLOY_STRATEGY template: metadata: labels: @@ -61,6 +61,10 @@ spec: readOnlyRootFilesystem: true runAsUser: 1000 runAsGroup: 1000 + resources: + requests: + cpu: 250m + memory: 4Gi # ----------------------------------------------------- # Nginx Container # ----------------------------------------------------- @@ -87,6 +91,10 @@ spec: # Replaces the original default.conf mountPath: /etc/nginx/conf.d/default.conf subPath: nginx-datatracker.conf + resources: + requests: + cpu: 10m + memory: 10Mi # ----------------------------------------------------- # ScoutAPM Container # ----------------------------------------------------- @@ -112,9 +120,14 @@ spec: readOnlyRootFilesystem: true runAsUser: 65534 # "nobody" user by default runAsGroup: 65534 # "nogroup" group by default + resources: + requests: + cpu: 10m + memory: 50Mi initContainers: - name: migration image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG" + imagePullPolicy: Always env: - name: "CONTAINER_ROLE" value: "migrations" @@ -159,8 +172,6 @@ spec: - name: nginx-tmp emptyDir: sizeLimit: "500Mi" - dnsPolicy: ClusterFirst - restartPolicy: Always terminationGracePeriodSeconds: 60 --- apiVersion: v1 diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml index 2b623da2bd..769cb03517 100644 --- a/k8s/kustomization.yaml +++ b/k8s/kustomization.yaml @@ -14,3 +14,4 @@ resources: - datatracker.yaml - memcached.yaml - rabbitmq.yaml + - replicator.yaml diff --git a/k8s/memcached.yaml b/k8s/memcached.yaml index 8f73f3d0d5..68b732d745 100644 --- a/k8s/memcached.yaml +++ b/k8s/memcached.yaml @@ -13,16 +13,6 @@ spec: labels: app: memcached spec: - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - datatracker - topologyKey: "kubernetes.io/hostname" securityContext: runAsNonRoot: true containers: @@ -46,6 +36,10 @@ spec: # memcached image sets up uid/gid 11211 runAsUser: 11211 runAsGroup: 11211 + resources: + requests: + cpu: 100m + memory: 100Mi # ----------------------------------------------------- # Memcached Exporter for Prometheus # ----------------------------------------------------- @@ -64,6 +58,10 @@ spec: readOnlyRootFilesystem: true runAsUser: 65534 # nobody runAsGroup: 65534 # nobody + resources: + requests: + cpu: 10m + memory: 20Mi dnsPolicy: ClusterFirst restartPolicy: Always terminationGracePeriodSeconds: 30 diff --git a/k8s/nginx-auth.conf b/k8s/nginx-auth.conf index 95aa838064..1dc2f4fae0 100644 --- a/k8s/nginx-auth.conf +++ b/k8s/nginx-auth.conf @@ -1,3 +1,7 @@ +upstream datatracker_backend { + server 127.0.0.1:8000; +} + server { listen 8080 default_server; server_name _; @@ -33,7 +37,7 @@ server { proxy_set_header X-Request-Start "t=$${keepempty}msec"; proxy_set_header X-Forwarded-For $${keepempty}proxy_add_x_forwarded_for; proxy_hide_header X-Datatracker-Is-Authenticated; # hide this from the outside world - proxy_pass http://localhost:8000; + proxy_pass http://datatracker_backend; # Set timeouts longer than Cloudflare proxy limits proxy_connect_timeout 60; # nginx default (Cf = 15) proxy_read_timeout 120; # nginx default = 60 (Cf = 100) diff --git a/k8s/nginx-datatracker.conf b/k8s/nginx-datatracker.conf index 882d7563c2..a7d5936a19 100644 --- a/k8s/nginx-datatracker.conf +++ b/k8s/nginx-datatracker.conf @@ -1,3 +1,7 @@ +upstream datatracker_backend { + server 127.0.0.1:8000; +} + server { listen 8080 default_server; server_name _; @@ -22,7 +26,7 @@ server { proxy_set_header X-Request-Start "t=$${keepempty}msec"; proxy_set_header X-Forwarded-For $${keepempty}proxy_add_x_forwarded_for; proxy_hide_header X-Datatracker-Is-Authenticated; # hide this from the outside world - proxy_pass http://localhost:8000; + proxy_pass http://datatracker_backend; # Set timeouts longer than Cloudflare proxy limits proxy_connect_timeout 60; # nginx default (Cf = 15) proxy_read_timeout 120; # nginx default = 60 (Cf = 100) diff --git a/k8s/rabbitmq.yaml b/k8s/rabbitmq.yaml index 0c8f0705b5..e69aa7a1aa 100644 --- a/k8s/rabbitmq.yaml +++ b/k8s/rabbitmq.yaml @@ -13,16 +13,6 @@ spec: labels: app: rabbitmq spec: - affinity: - podAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchExpressions: - - key: app - operator: In - values: - - datatracker - topologyKey: "kubernetes.io/hostname" securityContext: runAsNonRoot: true containers: @@ -52,17 +42,17 @@ spec: key: CELERY_PASSWORD livenessProbe: exec: - command: ["rabbitmq-diagnostics", "-q", "ping"] + command: ["rabbitmq-diagnostics", "-q", "ping", "-t", "30"] periodSeconds: 30 - timeoutSeconds: 5 + timeoutSeconds: 35 # slightly longer than ping "-t" option startupProbe: initialDelaySeconds: 15 periodSeconds: 5 - timeoutSeconds: 5 + timeoutSeconds: 35 # slightly longer than ping "-t" option successThreshold: 1 failureThreshold: 60 exec: - command: ["rabbitmq-diagnostics", "-q", "ping"] + command: ["rabbitmq-diagnostics", "-q", "ping", "-t", "30"] securityContext: allowPrivilegeEscalation: false capabilities: @@ -72,6 +62,10 @@ spec: # rabbitmq image sets up uid/gid 100/101 runAsUser: 100 runAsGroup: 101 + resources: + requests: + cpu: 100m + memory: 150Mi initContainers: # ----------------------------------------------------- # Init RabbitMQ data diff --git a/k8s/replicator.yaml b/k8s/replicator.yaml new file mode 100644 index 0000000000..0b06fe4fdc --- /dev/null +++ b/k8s/replicator.yaml @@ -0,0 +1,76 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: replicator + labels: + deleteBeforeUpgrade: yes +spec: + replicas: 1 + revisionHistoryLimit: 2 + selector: + matchLabels: + app: replicator + strategy: + type: Recreate + template: + metadata: + labels: + app: replicator + spec: + securityContext: + runAsNonRoot: true + containers: + # ----------------------------------------------------- + # Celery Container + # ----------------------------------------------------- + - name: celery + image: "ghcr.io/ietf-tools/datatracker:$APP_IMAGE_TAG" + imagePullPolicy: Always + volumeMounts: + - name: dt-vol + mountPath: /a + - name: dt-tmp + mountPath: /tmp + - name: dt-home + mountPath: /home/datatracker + - name: dt-xml2rfc-cache + mountPath: /var/cache/xml2rfc + - name: dt-cfg + mountPath: /workspace/ietf/settings_local.py + subPath: settings_local.py + env: + - name: "CONTAINER_ROLE" + value: "replicator" + envFrom: + - secretRef: + name: dt-secrets-env + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsUser: 1000 + runAsGroup: 1000 + resources: + requests: + cpu: 100m + memory: 500Mi + volumes: + # To be overridden with the actual shared volume + - name: dt-vol + - name: dt-tmp + emptyDir: + sizeLimit: "2Gi" + - name: dt-xml2rfc-cache + emptyDir: + sizeLimit: "2Gi" + - name: dt-home + emptyDir: + sizeLimit: "2Gi" + - name: dt-cfg + configMap: + name: files-cfgmap + dnsPolicy: ClusterFirst + restartPolicy: Always + terminationGracePeriodSeconds: 600 diff --git a/k8s/secrets.yaml b/k8s/secrets.yaml index ba90af9c2a..4d32652146 100644 --- a/k8s/secrets.yaml +++ b/k8s/secrets.yaml @@ -39,7 +39,6 @@ stringData: DATATRACKER_NOMCOM_APP_SECRET_B64: "m9pzMezVoFNJfsvU9XSZxGnXnwup6P5ZgCQeEnROOoQ=" # secret DATATRACKER_IANA_SYNC_PASSWORD: "this-is-the-iana-sync-password" # secret - DATATRACKER_RFC_EDITOR_SYNC_PASSWORD: "this-is-the-rfc-editor-sync-password" # secret DATATRACKER_YOUTUBE_API_KEY: "this-is-the-youtube-api-key" # secret DATATRACKER_GITHUB_BACKUP_API_KEY: "this-is-the-github-backup-api-key" # secret @@ -80,4 +79,4 @@ stringData: # Scout configuration DATATRACKER_SCOUT_KEY: "this-is-the-scout-key" - DATATRACKER_SCOUT_NAME: "StagingDatatracker" \ No newline at end of file + DATATRACKER_SCOUT_NAME: "StagingDatatracker" diff --git a/k8s/settings_local.py b/k8s/settings_local.py index a0e197b7f5..402f89787b 100644 --- a/k8s/settings_local.py +++ b/k8s/settings_local.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2007-2024, All Rights Reserved +# Copyright The IETF Trust 2007-2026, All Rights Reserved # -*- coding: utf-8 -*- from base64 import b64decode @@ -7,25 +7,24 @@ from ietf import __release_hash__ from ietf.settings import * # pyflakes:ignore -from ietf.settings import STORAGES, MORE_STORAGE_NAMES, BLOBSTORAGE_CONNECT_TIMEOUT, BLOBSTORAGE_READ_TIMEOUT, BLOBSTORAGE_MAX_ATTEMPTS +from ietf.settings import ( + STORAGES, + ARTIFACT_STORAGE_NAMES, + BLOBSTORAGE_CONNECT_TIMEOUT, + BLOBSTORAGE_READ_TIMEOUT, + BLOBSTORAGE_MAX_ATTEMPTS, +) import botocore.config def _multiline_to_list(s): - """Helper to split at newlines and conver to list""" + """Helper to split at newlines and convert to list""" return [item.strip() for item in s.split("\n")] # Default to "development". Production _must_ set DATATRACKER_SERVER_MODE="production" in the env! SERVER_MODE = os.environ.get("DATATRACKER_SERVER_MODE", "development") -# Use X-Forwarded-Proto to determine request.is_secure(). This relies on CloudFlare overwriting the -# value of the header if an incoming request sets it, which it does: -# https://developers.cloudflare.com/fundamentals/reference/http-request-headers/#x-forwarded-proto -# See also, especially the warnings: -# https://docs.djangoproject.com/en/dev/ref/settings/#secure-proxy-ssl-header -SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") - # Secrets _SECRET_KEY = os.environ.get("DATATRACKER_DJANGO_SECRET_KEY", None) if _SECRET_KEY is not None: @@ -45,12 +44,6 @@ def _multiline_to_list(s): else: raise RuntimeError("DATATRACKER_IANA_SYNC_PASSWORD must be set") -_RFC_EDITOR_SYNC_PASSWORD = os.environ.get("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD", None) -if _RFC_EDITOR_SYNC_PASSWORD is not None: - RFC_EDITOR_SYNC_PASSWORD = os.environ.get("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD") -else: - raise RuntimeError("DATATRACKER_RFC_EDITOR_SYNC_PASSWORD must be set") - _YOUTUBE_API_KEY = os.environ.get("DATATRACKER_YOUTUBE_API_KEY", None) if _YOUTUBE_API_KEY is not None: YOUTUBE_API_KEY = _YOUTUBE_API_KEY @@ -81,6 +74,22 @@ def _multiline_to_list(s): else: raise RuntimeError("DATATRACKER_API_PRIVATE_KEY_PEM_B64 must be set") +_RED_PRECOMPUTER_TRIGGER_RETRY_DELAY = os.environ.get( + "DATATRACKER_RED_PRECOMPUTER_TRIGGER_RETRY_DELAY", None +) +if _RED_PRECOMPUTER_TRIGGER_RETRY_DELAY is not None: + RED_PRECOMPUTER_TRIGGER_RETRY_DELAY = _RED_PRECOMPUTER_TRIGGER_RETRY_DELAY +_RED_PRECOMPUTER_TRIGGER_MAX_RETRIES = os.environ.get( + "DATATRACKER_RED_PRECOMPUTER_TRIGGER_MAX_RETRIES", None +) +if _RED_PRECOMPUTER_TRIGGER_MAX_RETRIES is not None: + RED_PRECOMPUTER_TRIGGER_MAX_RETRIES = _RED_PRECOMPUTER_TRIGGER_MAX_RETRIES +_TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL = os.environ.get( + "DATATRACKER_TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL", None +) +if _TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL is not None: + TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL = _TRIGGER_RED_PRECOMPUTE_MULTIPLE_URL + # Set DEBUG if DATATRACKER_DEBUG env var is the word "true" DEBUG = os.environ.get("DATATRACKER_DEBUG", "false").lower() == "true" @@ -99,19 +108,33 @@ def _multiline_to_list(s): "PASSWORD": os.environ.get("DATATRACKER_DB_PASS", ""), "OPTIONS": json.loads(os.environ.get("DATATRACKER_DB_OPTS_JSON", "{}")), }, + "blobdb": { + "HOST": os.environ.get("BLOBDB_DB_HOST", "blobdb"), + "PORT": os.environ.get("BLOBDB_DB_PORT", "5432"), + "NAME": os.environ.get("BLOBDB_DB_NAME", "blob"), + "ENGINE": "django.db.backends.postgresql", + "USER": os.environ.get("BLOBDB_DB_USER", "django"), + "PASSWORD": os.environ.get("BLOBDB_DB_PASS", ""), + "OPTIONS": json.loads(os.environ.get("BLOBDB_DB_OPTS_JSON", "{}")), + }, } +DATABASE_ROUTERS = ["ietf.blobdb.routers.BlobdbStorageRouter"] +BLOBDB_DATABASE = "blobdb" + # Configure persistent connections. A setting of 0 is Django's default. _conn_max_age = os.environ.get("DATATRACKER_DB_CONN_MAX_AGE", "0") -# A string "none" means unlimited age. -DATABASES["default"]["CONN_MAX_AGE"] = ( - None if _conn_max_age.lower() == "none" else int(_conn_max_age) -) +for dbname in ["default", "blobdb"]: + # A string "none" means unlimited age. + DATABASES[dbname]["CONN_MAX_AGE"] = ( + None if _conn_max_age.lower() == "none" else int(_conn_max_age) + ) # Enable connection health checks if DATATRACKER_DB_CONN_HEALTH_CHECK is the string "true" _conn_health_checks = bool( os.environ.get("DATATRACKER_DB_CONN_HEALTH_CHECKS", "false").lower() == "true" ) -DATABASES["default"]["CONN_HEALTH_CHECKS"] = _conn_health_checks +for dbname in ["default", "blobdb"]: + DATABASES[dbname]["CONN_HEALTH_CHECKS"] = _conn_health_checks # DATATRACKER_ADMINS is a newline-delimited list of addresses parseable by email.utils.parseaddr _admins_str = os.environ.get("DATATRACKER_ADMINS", None) @@ -126,14 +149,26 @@ def _multiline_to_list(s): EMAIL_HOST = os.environ.get("DATATRACKER_EMAIL_HOST", "localhost") EMAIL_PORT = int(os.environ.get("DATATRACKER_EMAIL_PORT", "2025")) +_broker_url = os.environ.get("DATATRACKER_BROKER_URL", None) _celery_password = os.environ.get("CELERY_PASSWORD", None) -if _celery_password is None: - raise RuntimeError("CELERY_PASSWORD must be set") -CELERY_BROKER_URL = "amqp://datatracker:{password}@{host}/{queue}".format( - host=os.environ.get("RABBITMQ_HOSTNAME", "dt-rabbitmq"), - password=_celery_password, - queue=os.environ.get("RABBITMQ_QUEUE", "dt"), +if _broker_url is not None: + CELERY_BROKER_URL = _broker_url +elif _celery_password is not None: + CELERY_BROKER_URL = "amqp://datatracker:{password}@{host}/{queue}".format( + host=os.environ.get("RABBITMQ_HOSTNAME", "dt-rabbitmq"), + password=_celery_password, + queue=os.environ.get("RABBITMQ_QUEUE", "dt"), + ) +else: + raise RuntimeError("DATATRACKER_BROKER_URL or CELERY_PASSWORD must be set") + +# mailarchive API key +_mailing_list_archive_api_key = os.environ.get( + "DATATRACKER_MAILING_LIST_ARCHIVE_API_KEY", None ) +if _mailing_list_archive_api_key is None: + raise RuntimeError("DATATRACKER_MAILING_LIST_ARCHIVE_API_KEY must be set") +MAILING_LIST_ARCHIVE_API_KEY = _mailing_list_archive_api_key IANA_SYNC_USERNAME = "ietfsync" IANA_SYNC_CHANGES_URL = "https://datatracker.iana.org:4443/data-tracker/changes" @@ -146,6 +181,21 @@ def _multiline_to_list(s): raise RuntimeError("DATATRACKER_REGISTRATION_API_KEY must be set") STATS_REGISTRATION_ATTENDEES_JSON_URL = f"https://registration.ietf.org/{{number}}/attendees/?apikey={_registration_api_key}" +# Registration Participants API config - key must be set, but the URL can be left +# to the default in settings.py +_registration_participants_api_key = os.environ.get( + "DATATRACKER_REGISTRATION_PARTICIPANTS_API_KEY", None +) +if _registration_participants_api_key is None: + raise RuntimeError("DATATRACKER_REGISTRATION_PARTICIPANTS_API_KEY must be set") +REGISTRATION_PARTICIPANTS_API_KEY = _registration_participants_api_key + +_registration_participants_api_url = os.environ.get( + "DATATRACKER_REGISTRATION_PARTICIPANTS_API_URL", None +) +if _registration_participants_api_url is not None: + REGISTRATION_PARTICIPANTS_API_URL = _registration_participants_api_url + # FIRST_CUTOFF_DAYS = 12 # SECOND_CUTOFF_DAYS = 12 # SUBMISSION_CUTOFF_DAYS = 26 @@ -173,7 +223,13 @@ def _multiline_to_list(s): "DATATRACKER_MEETECHO_CLIENT_ID and DATATRACKER_MEETECHO_CLIENT_SECRET must be set" ) -# For APP_API_TOKENS, ccept either base64-encoded JSON or raw JSON, but not both +# For APP_API_TOKENS, accept either base64-encoded JSON or raw JSON, but not both. +# To decode / pretty-print the encoded form, run: +# base64 -d | jq . +# paste the encoded secret into stdin. Copy/paste that into an editor you trust not +# to leave a copy lying around. When done editing, copy/paste the final JSON through +# jq -c | base64 +# and copy/paste the output into the secret store. if "DATATRACKER_APP_API_TOKENS_JSON_B64" in os.environ: if "DATATRACKER_APP_API_TOKENS_JSON" in os.environ: raise RuntimeError( @@ -240,7 +296,9 @@ def _multiline_to_list(s): PHOTOS_DIR = MEDIA_ROOT + PHOTOS_DIRNAME # Normally only set for debug, but needed until we have a real FS -DJANGO_VITE_MANIFEST_PATH = os.path.join(BASE_DIR, "static/dist-neue/manifest.json") +DJANGO_VITE["default"]["manifest_path"] = os.path.join( + BASE_DIR, "static/dist-neue/manifest.json" +) # Binaries that are different in the docker image DE_GFM_BINARY = "/usr/local/bin/de-gfm" @@ -257,6 +315,27 @@ def _multiline_to_list(s): "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", "VERSION": __version__, "KEY_PREFIX": "ietf:dt", + # Key function is default except with sha384-encoded key + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, + "agenda": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt:agenda", + # Key function is default except with sha384-encoded key + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, + "proceedings": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt:proceedings", + # Key function is default except with sha384-encoded key "KEY_FUNCTION": lambda key, key_prefix, version: ( f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" ), @@ -302,7 +381,7 @@ def _multiline_to_list(s): # Console logs as JSON instead of plain when running in k8s LOGGING["handlers"]["console"]["formatter"] = "json" -# Configure storages for the blob store +# Configure storages for the replica blob store _blob_store_endpoint_url = os.environ.get("DATATRACKER_BLOB_STORE_ENDPOINT_URL") _blob_store_access_key = os.environ.get("DATATRACKER_BLOB_STORE_ACCESS_KEY") _blob_store_secret_key = os.environ.get("DATATRACKER_BLOB_STORE_SECRET_KEY") @@ -311,9 +390,8 @@ def _multiline_to_list(s): "All of DATATRACKER_BLOB_STORE_ENDPOINT_URL, DATATRACKER_BLOB_STORE_ACCESS_KEY, " "and DATATRACKER_BLOB_STORE_SECRET_KEY must be set" ) -_blob_store_bucket_prefix = os.environ.get( - "DATATRACKER_BLOB_STORE_BUCKET_PREFIX", "" -) +_blob_store_bucket_prefix = os.environ.get("DATATRACKER_BLOB_STORE_BUCKET_PREFIX", "") +_blob_store_bucket_suffix = os.environ.get("DATATRACKER_BLOB_STORE_BUCKET_SUFFIX", "") _blob_store_enable_profiling = ( os.environ.get("DATATRACKER_BLOB_STORE_ENABLE_PROFILING", "false").lower() == "true" ) @@ -321,26 +399,120 @@ def _multiline_to_list(s): os.environ.get("DATATRACKER_BLOB_STORE_MAX_ATTEMPTS", BLOBSTORAGE_MAX_ATTEMPTS) ) _blob_store_connect_timeout = float( - os.environ.get("DATATRACKER_BLOB_STORE_CONNECT_TIMEOUT", BLOBSTORAGE_CONNECT_TIMEOUT) + os.environ.get( + "DATATRACKER_BLOB_STORE_CONNECT_TIMEOUT", BLOBSTORAGE_CONNECT_TIMEOUT + ) ) _blob_store_read_timeout = float( os.environ.get("DATATRACKER_BLOB_STORE_READ_TIMEOUT", BLOBSTORAGE_READ_TIMEOUT) ) -for storage_name in MORE_STORAGE_NAMES: - STORAGES[storage_name] = { - "BACKEND": "ietf.doc.storage_backends.CustomS3Storage", + +for storagename in ARTIFACT_STORAGE_NAMES: + if storagename in ["staging"]: + continue + replica_storagename = f"r2-{storagename}" + adjusted_bucket_name = ( + _blob_store_bucket_prefix + storagename + _blob_store_bucket_suffix + ).strip() + STORAGES[replica_storagename] = { + "BACKEND": "ietf.doc.storage.MetadataS3Storage", "OPTIONS": dict( endpoint_url=_blob_store_endpoint_url, access_key=_blob_store_access_key, secret_key=_blob_store_secret_key, security_token=None, client_config=botocore.config.Config( + request_checksum_calculation="when_required", + response_checksum_validation="when_required", signature_version="s3v4", connect_timeout=_blob_store_connect_timeout, read_timeout=_blob_store_read_timeout, retries={"total_max_attempts": _blob_store_max_attempts}, ), - bucket_name=f"{_blob_store_bucket_prefix}{storage_name}".strip(), + verify=False, + bucket_name=adjusted_bucket_name, ietf_log_blob_timing=_blob_store_enable_profiling, ), } + +# Configure storage for the red bucket - assume it uses the same credentials as +# other blobs +_red_bucket_name = os.environ.get("DATATRACKER_BLOB_STORE_RED_BUCKET_NAME", "").strip() +if _red_bucket_name == "": + raise RuntimeError("DATATRACKER_BLOB_STORE_RED_BUCKET_NAME must be set") + +STORAGES["red_bucket"] = { + "BACKEND": "storages.backends.s3.S3Storage", + "OPTIONS": dict( + endpoint_url=_blob_store_endpoint_url, + access_key=_blob_store_access_key, + secret_key=_blob_store_secret_key, + security_token=None, + client_config=botocore.config.Config( + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + signature_version="s3v4", + connect_timeout=_blob_store_connect_timeout, + read_timeout=_blob_store_read_timeout, + retries={"total_max_attempts": _blob_store_max_attempts}, + ), + verify=False, + bucket_name=_red_bucket_name, + ), +} +RFCINDEX_DELETE_THEN_WRITE = False # S3Storage allows file_overwrite by default +RFCINDEX_OUTPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_OUTPUT_PATH", "other/") +RFCINDEX_INPUT_PATH = os.environ.get("DATATRACKER_RFCINDEX_INPUT_PATH", "") + +# Configure the blobdb app for artifact storage +_blobdb_replication_enabled = ( + os.environ.get("DATATRACKER_BLOBDB_REPLICATION_ENABLED", "true").lower() == "true" +) +_blobdb_replication_verbose_logging = ( + os.environ.get("DATATRACKER_BLOBDB_REPLICATION_VERBOSE_LOGGING", "false").lower() + == "true" +) + +BLOBDB_REPLICATION = { + "ENABLED": _blobdb_replication_enabled, + "DEST_STORAGE_PATTERN": "r2-{bucket}", + "INCLUDE_BUCKETS": ARTIFACT_STORAGE_NAMES, + "EXCLUDE_BUCKETS": ["staging"], + "VERBOSE_LOGGING": _blobdb_replication_verbose_logging, +} + +# Optionally disable password strength enforcement at login (on by default) +PASSWORD_POLICY_ENFORCE_AT_LOGIN = ( + os.environ.get("DATATRACKER_ENFORCE_PW_POLICY", "true").lower() != "false" +) + +# Typesense search indexing +SEARCHINDEX_CONFIG = { + "TYPESENSE_API_URL": os.environ.get("DATATRACKER_TYPESENSE_API_URL", ""), + "TYPESENSE_API_KEY": os.environ.get("DATATRACKER_TYPESENSE_API_KEY", ""), + "TASK_RETRY_DELAY": int( + os.environ.get("DATATRACKER_SEARCHINDEX_TASK_RETRY_DELAY", "10") + ), + "TASK_MAX_RETRIES": int( + os.environ.get("DATATRACKER_SEARCHINDEX_TASK_MAX_RETRIES", "12") + ), +} + +# Errata system api configuration +ERRATA_METADATA_NOTIFICATION_API_KEY = os.environ.get( + "DATATRACKER_ERRATA_METADATA_NOTIFICATION_API_KEY", None +) +if ERRATA_METADATA_NOTIFICATION_API_KEY is not None: + ERRATA_METADATA_NOTIFICATION_URL = os.environ.get( + "DATATRACKER_ERRATA_METADATA_NOTIFICATION_URL", None + ) + if ERRATA_METADATA_NOTIFICATION_URL is None: + raise RuntimeError( + "DATATRACKER_ERRATA_METADATA_NOTIFICATION_URL must be set if " + "DATATRACKER_ERRATA_METADATA_NOTIFICATION_API_KEY is provided" + ) + +# name (with path) of errata.json in the red bucket +ERRATA_JSON_BLOB_NAME = os.environ.get( + "DATATRACKER_ERRATA_JSON_BLOB_NAME", "other/errata.json" +) diff --git a/mypy.ini b/mypy.ini index 19df7ec9b0..4acaf98c95 100644 --- a/mypy.ini +++ b/mypy.ini @@ -2,6 +2,9 @@ ignore_missing_imports = True +# allow PEP 695 type aliases (flag needed until mypy >= 1.13) +enable_incomplete_feature = NewGenericSyntax + plugins = mypy_django_plugin.main diff --git a/package.json b/package.json index 6d1591ca15..29ead19d23 100644 --- a/package.json +++ b/package.json @@ -16,12 +16,16 @@ "@fullcalendar/luxon3": "6.1.11", "@fullcalendar/timegrid": "6.1.11", "@fullcalendar/vue3": "6.1.11", + "@kurkle/color": "0.3.1", "@popperjs/core": "2.11.8", "@twuni/emojify": "1.0.2", "bootstrap": "5.3.3", "bootstrap-icons": "1.11.3", "browser-fs-access": "0.35.0", "caniuse-lite": "1.0.30001603", + "chart.js": "^4.5.1", + "chartjs-plugin-autocolors": "0.3.1", + "chartjs-plugin-zoom": "2.2.0", "d3": "7.9.0", "file-saver": "2.0.5", "highcharts": "11.4.0", @@ -112,11 +116,14 @@ "ietf/static/images/irtf-logo-white.svg", "ietf/static/images/irtf-logo.svg", "ietf/static/js/add_session_recordings.js", + "ietf/static/js/attendees-chart.js", "ietf/static/js/agenda_filter.js", "ietf/static/js/agenda_materials.js", + "ietf/static/js/announcement.js", "ietf/static/js/complete-review.js", "ietf/static/js/create_timeslot.js", "ietf/static/js/create_timeslot.js", + "ietf/static/js/custom_striped.js", "ietf/static/js/d3.js", "ietf/static/js/datepicker.js", "ietf/static/js/doc-search.js", @@ -143,10 +150,16 @@ "ietf/static/js/manage-community-list.js", "ietf/static/js/manage-review-requests.js", "ietf/static/js/meeting-interim-request.js", + "ietf/static/js/meeting_stats.js", + "ietf/static/js/meeting_timeline.js", "ietf/static/js/moment.js", + "ietf/static/js/navbar-doc-search.js", "ietf/static/js/password_strength.js", "ietf/static/js/select2.js", + "ietf/static/js/session_details.js", "ietf/static/js/session_details_form.js", + "ietf/static/js/session_form.js", + "ietf/static/js/session_request.js", "ietf/static/js/sortable.js", "ietf/static/js/stats.js", "ietf/static/js/status-change-edit-relations.js", @@ -207,8 +220,6 @@ "ietf/secr/static/images/tooltag-arrowright.webp", "ietf/secr/static/images/tooltag-arrowright_over.webp", "ietf/secr/static/js/dynamic_inlines.js", - "ietf/secr/static/js/session_form.js", - "ietf/secr/static/js/sessions.js", "ietf/secr/static/js/utils.js" ] } diff --git a/patch/django-cookie-delete-with-all-settings.patch b/patch/django-cookie-delete-settings-and-CVE-2026-35192.patch similarity index 63% rename from patch/django-cookie-delete-with-all-settings.patch rename to patch/django-cookie-delete-settings-and-CVE-2026-35192.patch index fb8bbbe4fe..3f625c33bb 100644 --- a/patch/django-cookie-delete-with-all-settings.patch +++ b/patch/django-cookie-delete-settings-and-CVE-2026-35192.patch @@ -9,9 +9,9 @@ samesite=settings.SESSION_COOKIE_SAMESITE, ) ---- django/http/response.py.orig 2020-08-13 11:16:04.060627793 +0200 -+++ django/http/response.py 2020-08-13 11:54:03.482476973 +0200 -@@ -282,20 +282,28 @@ +--- django/http/response.py.orig 2025-12-02 22:12:05.197283001 +0000 ++++ django/http/response.py 2025-12-02 22:26:01.396576013 +0000 +@@ -286,20 +286,28 @@ value = signing.get_cookie_signer(salt=key + salt).sign(value) return self.set_cookie(key, value, **kwargs) @@ -44,9 +44,9 @@ expires="Thu, 01 Jan 1970 00:00:00 GMT", samesite=samesite, ) ---- django/contrib/sessions/middleware.py.orig 2020-08-13 12:12:12.401898114 +0200 -+++ django/contrib/sessions/middleware.py 2020-08-13 12:14:52.690520659 +0200 -@@ -38,6 +38,8 @@ +--- django/contrib/sessions/middleware.py.old 2026-05-12 15:18:07.673997003 +0000 ++++ django/contrib/sessions/middleware.py 2026-05-12 15:18:15.770997007 +0000 +@@ -38,12 +38,15 @@ settings.SESSION_COOKIE_NAME, path=settings.SESSION_COOKIE_PATH, domain=settings.SESSION_COOKIE_DOMAIN, @@ -54,4 +54,23 @@ + httponly=settings.SESSION_COOKIE_HTTPONLY or None, samesite=settings.SESSION_COOKIE_SAMESITE, ) - patch_vary_headers(response, ("Cookie",)) +- patch_vary_headers(response, ("Cookie",)) ++ need_vary_cookie = True + else: +- if accessed: +- patch_vary_headers(response, ("Cookie",)) ++ # If the session was accessed, it must be varied on, regardless of ++ # whether it was modified or will be saved. ++ need_vary_cookie = accessed + if (modified or settings.SESSION_SAVE_EVERY_REQUEST) and not empty: + if request.session.get_expire_at_browser_close(): + max_age = None +@@ -74,4 +77,8 @@ + httponly=settings.SESSION_COOKIE_HTTPONLY or None, + samesite=settings.SESSION_COOKIE_SAMESITE, + ) ++ # With a session cookie set, it must be varied on. ++ need_vary_cookie = True ++ if need_vary_cookie: ++ patch_vary_headers(response, ("Cookie",)) + return response diff --git a/patch/tastypie-django22-fielderror-response.patch b/patch/tastypie-django22-fielderror-response.patch index ffb152d319..3b4418fc66 100644 --- a/patch/tastypie-django22-fielderror-response.patch +++ b/patch/tastypie-django22-fielderror-response.patch @@ -1,5 +1,5 @@ ---- tastypie/resources.py.orig 2020-08-24 13:14:25.463166100 +0200 -+++ tastypie/resources.py 2020-08-24 13:15:55.133759224 +0200 +--- tastypie/resources.py.orig 2025-07-29 19:00:01.526948002 +0000 ++++ tastypie/resources.py 2025-07-29 19:07:15.324127008 +0000 @@ -12,7 +12,7 @@ ObjectDoesNotExist, MultipleObjectsReturned, ValidationError, FieldDoesNotExist ) @@ -9,13 +9,13 @@ from django.db.models.fields.related import ForeignKey from django.urls.conf import re_path from tastypie.utils.timezone import make_naive_utc -@@ -2198,6 +2198,8 @@ +@@ -2216,6 +2216,8 @@ return self.authorized_read_list(objects, bundle) except ValueError: raise BadRequest("Invalid resource lookup data provided (mismatched type).") + except FieldError as e: + raise BadRequest("Invalid resource lookup: %s." % e) - + def obj_get(self, bundle, **kwargs): """ --- tastypie/paginator.py.orig 2020-08-25 15:24:46.391588425 +0200 diff --git a/playwright/.gitignore b/playwright/.gitignore index 75e854d8dc..f38d036a79 100644 --- a/playwright/.gitignore +++ b/playwright/.gitignore @@ -2,3 +2,4 @@ node_modules/ /test-results/ /playwright-report/ /playwright/.cache/ +auth.json \ No newline at end of file diff --git a/playwright/tests-legacy/secr/announcement.spec.js b/playwright/tests-legacy/secr/announcement.spec.js new file mode 100644 index 0000000000..4dbbc25a81 --- /dev/null +++ b/playwright/tests-legacy/secr/announcement.spec.js @@ -0,0 +1,77 @@ +const { test, expect } = require('@playwright/test') +const viewports = require('../../helpers/viewports') +const { setTimeout } = require('timers/promises') + +// ==================================================================== +// ANNOUNCEMENT | DESKTOP viewport +// ==================================================================== + +test.describe('desktop', () => { + + test.beforeAll(async ({ browser }) => { + const context = await browser.newContext(); + const page = await context.newPage(); + + await page.goto('/accounts/login/'); + + await page.fill('input#id_username', 'glen'); + await page.fill('input#id_password', 'password'); + + await page.click('button[type="submit"]'); + await page.waitForURL('/accounts/profile/'); + + await context.storageState({ path: 'auth.json' }); + + await context.close(); + }); + + test.beforeEach(async ({ browser }) => { + // Reuse the authentication state in each test + const context = await browser.newContext({ storageState: 'auth.json' }); + const page = await context.newPage(); + await page.setViewportSize({ + width: viewports.desktop[0], + height: viewports.desktop[1] + }) + await page.goto(`/secr/announcement/`); + await page.locator('h1:text("Announcement")').waitFor({ state: 'visible' }) + await setTimeout(500) + // Attach the page to the test context + test.info().page = page; + }) + + test('show to custom', async () => { + const page = test.info().page; + + // to_custom should initially be hidden + const element = page.locator('#id_to_custom'); + await expect(element).toBeHidden(); + await page.selectOption('select#id_to', 'Other...'); + await expect(element).toBeVisible(); + }) + + test('back button', async () => { + const page = test.info().page; + + const element = page.locator('#id_to_custom'); + await page.selectOption('select#id_to', 'Other...'); + await expect(element).toBeVisible(); + await page.fill('input#id_to_custom', 'custom@example.com'); + await page.selectOption('select#id_frm', 'IETF Chair '); + await page.fill('input#id_reply_to', 'greg@example.com'); + await page.fill('input#id_subject', 'About Stuff'); + await page.fill('textarea#id_body', 'This is the stuff'); + + await page.click('text="Continue"'); + const h2Locator = page.locator('h2:text("Confirm Announcement")'); + await h2Locator.waitFor({ state: 'visible' }); + + // click back button and check to_custom + await page.click('text="Back"'); + const subjectLocator = page.locator('input#id_subject'); + await subjectLocator.waitFor({ state: 'visible' }); + await expect(element).toBeVisible(); + await expect(element).toHaveValue('custom@example.com'); + }) + +}) \ No newline at end of file diff --git a/playwright/tests/meeting/agenda.spec.js b/playwright/tests/meeting/agenda.spec.js index 412a3fe9b8..2248027a38 100644 --- a/playwright/tests/meeting/agenda.spec.js +++ b/playwright/tests/meeting/agenda.spec.js @@ -1219,7 +1219,12 @@ test.describe('future - desktop', () => { await expect(eventButtons.locator(`#btn-lnk-${event.id}-calendar > i.bi`)).toBeVisible() } } else { - await expect(eventButtons).toHaveCount(0) + if (event.links.calendar) { + await expect(eventButtons.locator(`#btn-lnk-${event.id}-calendar`)).toHaveAttribute('href', event.links.calendar) + await expect(eventButtons.locator(`#btn-lnk-${event.id}-calendar > i.bi`)).toBeVisible() + } else { + await expect(eventButtons).toHaveCount(0) + } } } } diff --git a/requirements.txt b/requirements.txt index cd93f448e2..31e8ea69d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,85 +1,95 @@ # -*- conf-mode -*- -setuptools>=51.1.0 # Require this first, to prevent later errors +setuptools>=80.9.0 # Require this first, to prevent later errors # -argon2-cffi>=21.3.0 # For the Argon2 password hasher option -beautifulsoup4>=4.11.1 # Only used in tests -bibtexparser>=1.2.0 # Only used in tests -bleach>=6 -types-bleach>=6 -boto3>=1.35,<1.36 -boto3-stubs[s3]>=1.35,<1.36 -botocore>=1.35,<1.36 -celery>=5.2.6 -coverage>=4.5.4,<5.0 # Coverage 5.x moves from a json database to SQLite. Moving to 5.x will require substantial rewrites in ietf.utils.test_runner and ietf.release.views +aiosmtpd>=1.4.6 +argon2-cffi>=25.1.0 # For the Argon2 password hasher option +beautifulsoup4>=4.13.4 # Only used in tests +bibtexparser>=1.4.4 # Only used in tests +bleach>=6.2.0 # project is deprecated but supported +types-bleach>=6.2.0 +boto3>=1.39.15 +boto3-stubs[s3]>=1.39.15 +botocore>=1.39.15 +celery>=5.5.3 +coverage>=7.9.2 defusedxml>=0.7.1 # for TastyPie when using xml; not a declared dependency -Django>4.2,<5 -django-admin-rangefilter>=0.13.2 -django-analytical>=3.1.0 -django-bootstrap5>=21.3 -django-celery-beat>=2.3.0 -django-celery-results>=2.5.1 +Django>=4.2.30,<5 +django-admin-rangefilter>=0.13.3 +django-analytical>=3.2.0 +django-bootstrap5>=25.1 +django-celery-beat>=2.9.0 +django-celery-results>=2.6.0 django-csp>=3.7 -django-cors-headers>=3.11.0 -django-debug-toolbar>=3.2.4 -django-markup>=1.5 # Limited use - need to reconcile against direct use of markdown +django-cors-headers>=4.7.0 +django-debug-toolbar>=6.0.0 +django-filter>=24.3 +django-markup>=1.10 # Limited use - need to reconcile against direct use of markdown django-oidc-provider==0.8.2 # 0.8.3 changes logout flow and claim return -django-referrer-policy>=1.0 -django-simple-history>=3.0.0 -django-storages>=1.14.4 +django-simple-history>=3.10.1 +django-storages>=1.14.6 django-stubs>=4.2.7,<5 # The django-stubs version used determines the the mypy version indicated below -django-tastypie>=0.14.7,<0.15.0 # Version must be locked in sync with version of Django -django-vite>=2.0.2,<3 +django-tastypie>=0.15.1 # Version must be kept in sync with Django +django-vite>=3.1.0 django-widget-tweaks>=1.4.12 -djangorestframework>=3.15,<4 -djlint>=1.0.0 # To auto-indent templates via "djlint --profile django --reformat" -docutils>=0.18.1 # Used only by dbtemplates for RestructuredText +djangorestframework>=3.16.0 +docutils>=0.22.0 # Used only by dbtemplates for RestructuredText +types-docutils>=0.21.0 # should match docutils (0.22.0 not out yet) drf-spectacular>=0.27 -drf-standardized-errors[openapi] >= 0.14 -types-docutils>=0.18.1 -factory-boy>=3.3 -github3.py>=3.2.0 -gunicorn>=20.1.0 +drf-standardized-errors[openapi] >= 0.15.0 +factory-boy>=3.3.3 +gunicorn>=23.0.0 hashids>=1.3.1 -html2text>=2020.1.16 # Used only to clean comment field of secr/sreq +html2text>=2025.4.15 # Used only to clean comment field of secr/sreq html5lib>=1.1 # Only used in tests -inflect>= 6.0.2 -jsonfield>=3.1.0 # for SubmissionCheck. This is https://github.com/bradjasper/django-jsonfield/. -jsonschema[format]>=4.2.1 -jwcrypto>=1.2 # for signed notifications - this is aspirational, and is not really used. -logging_tree>=1.9 # Used only by the showloggers management command -lxml>=5.3.0 -markdown>=3.3.6 -types-markdown>=3.3.6 -mock>=4.0.3 # Used only by tests, of course -types-mock>=4.0.3 -mypy~=1.7.0 # Version requirements determined by django-stubs. -oic>=1.3 # Used only by tests -Pillow>=9.1.0 -psycopg2>=2.9.6 -pyang>=2.5.3 -pydyf>0.8.0 -pyflakes>=2.4.0 -pyopenssl>=22.0.0 # Used by urllib3.contrib, which is used by PyQuery but not marked as a dependency -pyquery>=1.4.3 -python-dateutil>=2.8.2 -types-python-dateutil>=2.8.2 -python-json-logger>=3.1.0 +httpx>=0.28.1 # Indirect req of typesense, but we import and refer to exceptions +icalendar>=5.0.0 +inflect>= 7.5.0 +jsonfield>=3.2.0 # deprecated - need to replace with Django's JSONField +jsonschema[format]>=4.25.0 +jwcrypto>=1.5.6 # for signed notifications - this is aspirational, and is not really used. +logging_tree>=1.10 # Used only by the showloggers management command +lxml>=6.0.0 +markdown>=3.8.0 +types-markdown>=3.8.0 +mock>=5.2.0 # should replace with unittest.mock and remove dependency +types-mock>=5.2.0 +mypy~=1.11.2 # Version requirements loosely determined by django-stubs. +oic>=1.7.0 # Used only by tests +opentelemetry-sdk>=1.38.0 +opentelemetry-instrumentation-django>=0.59b0 +opentelemetry-instrumentation-psycopg2>=0.59b0 +opentelemetry-instrumentation-pymemcache>=0.59b0 +opentelemetry-instrumentation-requests>=0.59b0 +opentelemetry-exporter-otlp-proto-http>=1.38.0 +pillow>=11.3.0 +psycopg2>=2.9.10 +pyang>=2.6.1 +pydyf>=0.11.0 +pyflakes>=3.4.0 +pyopenssl>=25.1.0 # Used by urllib3.contrib, which is used by PyQuery but not marked as a dependency +pyquery>=2.0.1 +python-dateutil>=2.9.0 +types-python-dateutil>=2.9.0 +python-json-logger>=3.3.0 python-magic==0.4.18 # Versions beyond the yanked .19 and .20 introduce form failures pymemcache>=4.0.0 # for django.core.cache.backends.memcached.PyMemcacheCache -python-mimeparse>=1.6 # from TastyPie -pytz==2022.2.1 # Pinned as changes need to be vetted for their effect on Meeting fields -types-pytz==2022.2.1 # match pytz version -requests>=2.31.0 -types-requests>=2.27.1 -requests-mock>=1.9.3 +python-mimeparse>=2.0.0 # from TastyPie +pytz==2025.2 # Pinned as changes need to be vetted for their effect on Meeting fields +types-pytz==2025.2.0.20251108 # match pytz version +typesense>=2.0.0 +requests>=2.32.4 +types-requests>=2.32.4 +requests-mock>=1.12.1 rfc2html>=2.0.3 -scout-apm>=2.24.2 -selenium>=4.0 -tblib>=1.7.0 # So that the django test runner provides tracebacks -tlds>=2022042700 # Used to teach bleach about which TLDs currently exist -tqdm>=4.64.0 -Unidecode>=1.3.4 -urllib3>=1.26,<2 -weasyprint>=64.1 -xml2rfc>=3.23.0 -xym>=0.6,<1.0 +scout-apm>=3.4.0 +selenium>=4.34.2 +tblib>=3.1.0 # So that the django test runner provides tracebacks +tlds>=2022042700 # Used to teach bleach about which TLDs currently exist +tqdm>=4.67.1 +unidecode>=1.4.0 +urllib3>=2.5.0 +weasyprint>=66.0 +xml2rfc>=3.30.0 +xym>=0.6,<0.10.0 +zxcvbn>=4.5.0 +types-zxcvbn~=4.5.0.20250223 # match zxcvbn version diff --git a/yarn.lock b/yarn.lock index 54768ac391..47d675d6b9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -532,6 +532,20 @@ __metadata: languageName: node linkType: hard +"@kurkle/color@npm:0.3.1": + version: 0.3.1 + resolution: "@kurkle/color@npm:0.3.1" + checksum: e6be5c081bf5acfd4a1803dcd5a0733caf450e73148d5f02dc536b1ff0c60c959c23472a26c9c3c6c78ada04fb6a53c9202db9b2de8ea56f6eeec381f9cc3a1a + languageName: node + linkType: hard + +"@kurkle/color@npm:^0.3.0": + version: 0.3.4 + resolution: "@kurkle/color@npm:0.3.4" + checksum: b95c6abe0241ba1745b3c84de3b464296b95ce577110b54f46e6c6dcc9a0966491533df43812bd6c66f92cf818e385d1390b280cd5851d4afb52fc37f8a6c0b9 + languageName: node + linkType: hard + "@lezer/common@npm:^0.15.0, @lezer/common@npm:^0.15.7": version: 0.15.12 resolution: "@lezer/common@npm:0.15.12" @@ -1944,6 +1958,13 @@ __metadata: languageName: node linkType: hard +"@types/hammerjs@npm:^2.0.45": + version: 2.0.46 + resolution: "@types/hammerjs@npm:2.0.46" + checksum: caba6ec788d19905c71092670b58514b3d1f5eee5382bf9205e8df688d51e7857b7994e2dd7aed57fac8977bdf0e456d67fbaf23440a4385b8ce25fe2af1ec39 + languageName: node + linkType: hard + "@types/istanbul-lib-coverage@npm:^2.0.1": version: 2.0.4 resolution: "@types/istanbul-lib-coverage@npm:2.0.4" @@ -2728,6 +2749,37 @@ browserlist@latest: languageName: node linkType: hard +"chart.js@npm:^4.5.1": + version: 4.5.1 + resolution: "chart.js@npm:4.5.1" + dependencies: + "@kurkle/color": ^0.3.0 + checksum: 34b35b373642994b2adac197e91363625930530e29fc1baa6dbb411b5e1295f9f6572922003a0224a21a3019aec916567c1ed00c33b1373081f189fc188e5a7b + languageName: node + linkType: hard + +"chartjs-plugin-autocolors@npm:0.3.1": + version: 0.3.1 + resolution: "chartjs-plugin-autocolors@npm:0.3.1" + peerDependencies: + "@kurkle/color": ^0.3.1 + chart.js: ">=2" + checksum: de4f87b5bb3e042aa1d3de3886425bbd2340a55ca455b645569d0def602079833182ef214e205ff4466fb5ab1e708761cf37eb51ab3cd622284242c05ed94128 + languageName: node + linkType: hard + +"chartjs-plugin-zoom@npm:2.2.0": + version: 2.2.0 + resolution: "chartjs-plugin-zoom@npm:2.2.0" + dependencies: + "@types/hammerjs": ^2.0.45 + hammerjs: ^2.0.8 + peerDependencies: + chart.js: ">=3.2.0" + checksum: a540e3834082eeb4dedb5ec6ca381f94d7e101075c19a7b65f2a4cd2d12685b3a416e718c9cf7145799802874fb397f69b71a955dfc56b035946cde4d1eb6c8e + languageName: node + linkType: hard + "chokidar@npm:>=3.0.0 <4.0.0": version: 3.5.3 resolution: "chokidar@npm:3.5.3" @@ -4616,6 +4668,13 @@ browserlist@latest: languageName: node linkType: hard +"hammerjs@npm:^2.0.8": + version: 2.0.8 + resolution: "hammerjs@npm:2.0.8" + checksum: b092da7d1565a165d7edb53ef0ce212837a8b11f897aa3cf81a7818b66686b0ab3f4747fbce8fc8a41d1376594639ce3a054b0fd4889ca8b5b136a29ca500e27 + languageName: node + linkType: hard + "has-bigints@npm:^1.0.1, has-bigints@npm:^1.0.2": version: 1.0.2 resolution: "has-bigints@npm:1.0.2" @@ -7030,6 +7089,7 @@ browserlist@latest: "@fullcalendar/luxon3": 6.1.11 "@fullcalendar/timegrid": 6.1.11 "@fullcalendar/vue3": 6.1.11 + "@kurkle/color": 0.3.1 "@parcel/optimizer-data-url": 2.12.0 "@parcel/transformer-inline-string": 2.12.0 "@parcel/transformer-sass": 2.12.0 @@ -7044,6 +7104,9 @@ browserlist@latest: browserlist: latest c8: 9.1.0 caniuse-lite: 1.0.30001603 + chart.js: ^4.5.1 + chartjs-plugin-autocolors: 0.3.1 + chartjs-plugin-zoom: 2.2.0 d3: 7.9.0 eslint: 8.57.0 eslint-config-standard: 17.1.0