diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 6b0fd79bb3..e4964e8909 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -23,7 +23,6 @@ "dbaeumer.vscode-eslint", "eamodio.gitlens", "editorconfig.editorconfig", - // Newer volar >=3.0.0 causes crashes in devcontainers "vue.volar@2.2.10", "mrmlnc.vscode-duplicate", "ms-azuretools.vscode-docker", @@ -33,9 +32,8 @@ "mutantdino.resourcemonitor", "oderwat.indent-rainbow", "redhat.vscode-yaml", - "spmeesseman.vscode-taskexplorer", - "visualstudioexptteam.vscodeintellicode", - "ms-python.pylint" + "ms-python.pylint", + "charliermarsh.ruff" ], "settings": { "terminal.integrated.defaultProfile.linux": "zsh", diff --git a/.devcontainer/docker-compose.extend.yml b/.devcontainer/docker-compose.extend.yml index a92f42bc6d..ce1ce259fd 100644 --- a/.devcontainer/docker-compose.extend.yml +++ b/.devcontainer/docker-compose.extend.yml @@ -14,8 +14,8 @@ services: network_mode: service:db blobstore: ports: - - '9000' - - '9001' + - '9000:9000' + - '9001:9001' volumes: datatracker-vscode-ext: diff --git a/.github/workflows/build-base-app.yml b/.github/workflows/build-base-app.yml index ef8a17f6b4..4a4394fca0 100644 --- a/.github/workflows/build-base-app.yml +++ b/.github/workflows/build-base-app.yml @@ -51,6 +51,7 @@ jobs: push: true tags: | ghcr.io/ietf-tools/datatracker-app-base:${{ env.IMGVERSION }} + ghcr.io/ietf-tools/datatracker-app-base:py312 ${{ github.ref == 'refs/heads/main' && 'ghcr.io/ietf-tools/datatracker-app-base:latest' || '' }} - name: Update version references diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8567446cae..d97889fbb8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,13 +16,13 @@ on: - Skip - Staging Only - Staging + Prod - sandbox: - description: 'Deploy to Sandbox' + dev: + description: 'Deploy to Dev' default: true required: true type: boolean - sandboxNoDbRefresh: - description: 'Sandbox Disable Daily DB Refresh' + devNoDbRefresh: + description: 'Dev Disable Daily DB Refresh' default: false required: true type: boolean @@ -170,7 +170,7 @@ jobs: fetch-tags: false - name: Setup Node.js environment - uses: actions/setup-node@v4 + uses: actions/setup-node@v6 with: node-version: 18.x @@ -278,8 +278,6 @@ jobs: tags: | ghcr.io/ietf-tools/datatracker:${{ env.PKG_VERSION }} ${{ env.FEATURE_LATEST_TAG && format('ghcr.io/ietf-tools/datatracker:{0}-latest', env.FEATURE_LATEST_TAG) || null }} - cache-from: type=gha - cache-to: type=gha,mode=max - name: Update CHANGELOG id: changelog @@ -392,44 +390,45 @@ jobs: value: "Failed" # ----------------------------------------------------------------- - # SANDBOX + # DEV # ----------------------------------------------------------------- - sandbox: - name: Deploy to Sandbox - if: ${{ !failure() && !cancelled() && github.event.inputs.sandbox == 'true' }} + dev: + name: Deploy to Dev + if: ${{ !failure() && !cancelled() && github.event.inputs.dev == 'true' }} needs: [prepare, release] - runs-on: [self-hosted, dev-server] + runs-on: ubuntu-latest environment: - name: sandbox + name: dev env: PKG_VERSION: ${{needs.prepare.outputs.pkg_version}} steps: - - uses: actions/checkout@v4 - - - name: Download a Release Artifact - uses: actions/download-artifact@v4.3.0 - with: - name: release-${{ env.PKG_VERSION }} - - - name: Deploy to containers - env: - DEBIAN_FRONTEND: noninteractive - run: | - echo "Reset production flags in settings.py..." - sed -i -r -e 's/^DEBUG *= *.*$/DEBUG = True/' -e "s/^SERVER_MODE *= *.*\$/SERVER_MODE = 'development'/" ietf/settings.py - echo "Install Deploy to Container CLI dependencies..." - cd dev/deploy-to-container - npm ci - cd ../.. - echo "Start Deploy..." - node ./dev/deploy-to-container/cli.js --branch ${{ github.ref_name }} --domain dev.ietf.org --appversion ${{ env.PKG_VERSION }} --commit ${{ github.sha }} --ghrunid ${{ github.run_id }} --nodbrefresh ${{ github.event.inputs.sandboxNoDbRefresh }} - - - name: Cleanup old docker resources - env: - DEBIAN_FRONTEND: noninteractive - run: | - docker image prune -a -f + - uses: actions/checkout@v4 + with: + ref: main + + - name: Get Deploy Name + env: + DEBIAN_FRONTEND: noninteractive + run: | + echo "Install Get Deploy Name CLI dependencies..." + cd dev/k8s-get-deploy-name + npm ci + echo "Get Deploy Name..." + echo "DEPLOY_NAMESPACE=$(node cli.js --branch ${{ github.ref_name }})" >> "$GITHUB_ENV" + + - name: Deploy to dev + uses: the-actions-org/workflow-dispatch@v4 + with: + workflow: deploy-dev.yml + repo: ietf-tools/infra-k8s + ref: main + token: ${{ secrets.GH_INFRA_K8S_TOKEN }} + inputs: '{ "app":"datatracker", "appVersion":"${{ env.PKG_VERSION }}", "remoteRef":"${{ github.sha }}", "namespace":"${{ env.DEPLOY_NAMESPACE }}", "disableDailyDbRefresh":${{ inputs.devNoDbRefresh }} }' + wait-for-completion: true + wait-for-completion-timeout: 60m + wait-for-completion-interval: 30s + display-workflow-run-url: false # ----------------------------------------------------------------- # STAGING @@ -452,9 +451,9 @@ jobs: repo: ietf-tools/infra-k8s ref: main token: ${{ secrets.GH_INFRA_K8S_TOKEN }} - inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "manifest":"postgres", "forceRecreate":true, "waitClusterReady":true }' + inputs: '{ "environment":"${{ secrets.GHA_K8S_CLUSTER }}", "app":"datatracker", "manifest":"postgres", "forceRecreate":true, "restoreToLastFullSnapshot":true, "waitClusterReady":true }' wait-for-completion: true - wait-for-completion-timeout: 30m + wait-for-completion-timeout: 120m wait-for-completion-interval: 20s display-workflow-run-url: false diff --git a/.github/workflows/dev-assets-sync-nightly.yml b/.github/workflows/dev-assets-sync-nightly.yml index 19933bddfd..4cfbf6365b 100644 --- a/.github/workflows/dev-assets-sync-nightly.yml +++ b/.github/workflows/dev-assets-sync-nightly.yml @@ -47,17 +47,3 @@ jobs: file: dev/shared-assets-sync/Dockerfile push: true tags: ghcr.io/ietf-tools/datatracker-rsync-assets:latest - - sync: - name: Run assets rsync - if: ${{ always() }} - runs-on: [self-hosted, dev-server] - needs: [build] - steps: - - name: Run rsync - env: - DEBIAN_FRONTEND: noninteractive - run: | - docker pull ghcr.io/ietf-tools/datatracker-rsync-assets:latest - docker run --rm -v dt-assets:/assets ghcr.io/ietf-tools/datatracker-rsync-assets:latest - docker image prune -a -f diff --git a/.github/workflows/sandbox-refresh.yml b/.github/workflows/sandbox-refresh.yml deleted file mode 100644 index 3ddb119e4f..0000000000 --- a/.github/workflows/sandbox-refresh.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Sandbox Refresh - -on: - # Run every night - schedule: - - cron: '0 9 * * *' - - workflow_dispatch: - -jobs: - main: - name: Refresh DBs - runs-on: [self-hosted, dev-server] - permissions: - contents: read - - steps: - - uses: actions/checkout@v4 - - - name: Refresh DBs - env: - DEBIAN_FRONTEND: noninteractive - run: | - echo "Install Deploy to Container CLI dependencies..." - cd dev/deploy-to-container - npm ci - cd ../.. - echo "Start Refresh..." - node ./dev/deploy-to-container/refresh.js - - - name: Cleanup old docker resources - env: - DEBIAN_FRONTEND: noninteractive - run: | - docker image prune -a -f diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f10c1db9a3..836314bac0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -104,7 +104,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v6 with: node-version: '18' diff --git a/.vscode/settings.json b/.vscode/settings.json index b0ceba5c9d..b323cd02f7 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -56,5 +56,6 @@ "python.linting.pylintArgs": ["--load-plugins", "pylint_django"], "python.testing.pytestEnabled": false, "python.testing.unittestEnabled": false, - "python.linting.enabled": true + "python.linting.enabled": true, + "python.terminal.shellIntegration.enabled": false } diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 4bd0b99363..8b36b0e6ac 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -105,10 +105,11 @@ "command": "/usr/local/bin/python", "args": [ "-m", - "smtpd", + "aiosmtpd", "-n", "-c", - "DebuggingServer", + "ietf.utils.aiosmtpd.DevDebuggingHandler", + "-l", "localhost:2025" ], "presentation": { diff --git a/README.md b/README.md index abebb7ca02..baffc311e7 100644 --- a/README.md +++ b/README.md @@ -5,10 +5,10 @@ [![Release](https://img.shields.io/github/release/ietf-tools/datatracker.svg?style=flat&maxAge=300)](https://github.com/ietf-tools/datatracker/releases) [![License](https://img.shields.io/github/license/ietf-tools/datatracker)](https://github.com/ietf-tools/datatracker/blob/main/LICENSE) [![Code Coverage](https://codecov.io/gh/ietf-tools/datatracker/branch/feat/bs5/graph/badge.svg?token=V4DXB0Q28C)](https://codecov.io/gh/ietf-tools/datatracker) -[![Python Version](https://img.shields.io/badge/python-3.9-blue?logo=python&logoColor=white)](#prerequisites) +[![Python Version](https://img.shields.io/badge/python-3.12-blue?logo=python&logoColor=white)](#prerequisites) [![Django Version](https://img.shields.io/badge/django-4.x-51be95?logo=django&logoColor=white)](#prerequisites) [![Node Version](https://img.shields.io/badge/node.js-16.x-green?logo=node.js&logoColor=white)](#prerequisites) -[![MariaDB Version](https://img.shields.io/badge/postgres-16-blue?logo=postgresql&logoColor=white)](#prerequisites) +[![MariaDB Version](https://img.shields.io/badge/postgres-17-blue?logo=postgresql&logoColor=white)](#prerequisites) ##### The day-to-day front-end to the IETF database for people who work on IETF standards. @@ -142,7 +142,7 @@ Pages will gradually be updated to Vue 3 components. These components are locate Each Vue 3 app has its own sub-directory. For example, the agenda app is located under `/client/agenda`. -The datatracker makes use of the Django-Vite plugin to point to either the Vite.js server or the precompiled production files. The `DJANGO_VITE_DEV_MODE` flag, found in the `ietf/settings_local.py` file determines whether the Vite.js server is used or not. +The datatracker makes use of the Django-Vite plugin to point to either the Vite.js server or the precompiled production files. The `DJANGO_VITE["default"]["dev_mode"]` flag, found in the `ietf/settings_local.py` file determines whether the Vite.js server is used or not. In development mode, you must start the Vite.js development server, in addition to the usual Datatracker server: diff --git a/client/agenda/AgendaDetailsModal.vue b/client/agenda/AgendaDetailsModal.vue index 2582bf2159..69c8ef8b53 100644 --- a/client/agenda/AgendaDetailsModal.vue +++ b/client/agenda/AgendaDetailsModal.vue @@ -274,6 +274,7 @@ async function fetchSessionMaterials () { diff --git a/client/agenda/AgendaScheduleList.vue b/client/agenda/AgendaScheduleList.vue index fc8b5fd30f..bbe5dfee8b 100644 --- a/client/agenda/AgendaScheduleList.vue +++ b/client/agenda/AgendaScheduleList.vue @@ -398,16 +398,6 @@ const meetingEvents = computed(() => { color: 'teal' }) } - // -> Calendar item - if (item.links.calendar) { - links.push({ - id: `lnk-${item.id}-calendar`, - label: 'Calendar (.ics) entry for this session', - icon: 'calendar-check', - href: item.links.calendar, - color: 'pink' - }) - } } else { // -> Post event if (meetingNumberInt >= 60) { @@ -484,6 +474,16 @@ const meetingEvents = computed(() => { } } } + // Add Calendar item for all events that has a calendar link + if (item.adjustedEnd > current && item.links.calendar) { + links.push({ + id: `lnk-${item.id}-calendar`, + label: 'Calendar (.ics) entry for this session', + icon: 'calendar-check', + href: item.links.calendar, + color: 'pink' + }) + } // Event icon let icon = null diff --git a/client/components/ChatLog.vue b/client/components/ChatLog.vue index f9dc382bfe..b3a4f7b40f 100644 --- a/client/components/ChatLog.vue +++ b/client/components/ChatLog.vue @@ -159,4 +159,18 @@ onMounted(() => { } } } + +[data-bs-theme="dark"] .chatlog { + .n-timeline-item-content__title { + color: #d63384 !important; + } + + .n-timeline-item-content__content { + color: #fff !important; + } + + .n-timeline-item-content__meta { + color: #0569ffd9 !important; + } +} diff --git a/client/components/Polls.vue b/client/components/Polls.vue index 30cc9e8f36..0846d4ed16 100644 --- a/client/components/Polls.vue +++ b/client/components/Polls.vue @@ -90,3 +90,21 @@ onMounted(() => { }) + diff --git a/dev/build/Dockerfile b/dev/build/Dockerfile index d619ee99ee..ce1828052e 100644 --- a/dev/build/Dockerfile +++ b/dev/build/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/ietf-tools/datatracker-app-base:20250821T1359 +FROM ghcr.io/ietf-tools/datatracker-app-base:20260304T1633 LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive diff --git a/dev/build/TARGET_BASE b/dev/build/TARGET_BASE index b6fc12e128..6be54fb6b0 100644 --- a/dev/build/TARGET_BASE +++ b/dev/build/TARGET_BASE @@ -1 +1 @@ -20250821T1359 +20260304T1633 diff --git a/dev/build/gunicorn.conf.py b/dev/build/gunicorn.conf.py index 032d95ee0d..9af4478685 100644 --- a/dev/build/gunicorn.conf.py +++ b/dev/build/gunicorn.conf.py @@ -1,4 +1,16 @@ -# Copyright The IETF Trust 2024, All Rights Reserved +# Copyright The IETF Trust 2024-2025, All Rights Reserved + +import os +import ietf +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.instrumentation.django import DjangoInstrumentor +from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor +from opentelemetry.instrumentation.pymemcache import PymemcacheInstrumentor +from opentelemetry.instrumentation.requests import RequestsInstrumentor # Configure security scheme headers for forwarded requests. Cloudflare sets X-Forwarded-Proto # for us. Don't trust any of the other similar headers. Only trust the header if it's coming @@ -119,3 +131,34 @@ def post_request(worker, req, environ, resp): in_flight = in_flight_by_pid.get(worker.pid, []) if request_description in in_flight: in_flight.remove(request_description) + +def post_fork(server, worker): + server.log.info("Worker spawned (pid: %s)", worker.pid) + + # Setting DATATRACKER_OPENTELEMETRY_ENABLE=all in the environment will enable all + # opentelemetry instrumentations. Individual instrumentations can be selected by + # using a space-separated list. See the code below for available instrumentations. + telemetry_env = os.environ.get("DATATRACKER_OPENTELEMETRY_ENABLE", "").strip() + if telemetry_env != "": + enabled_telemetry = [tok.strip().lower() for tok in telemetry_env.split()] + resource = Resource.create(attributes={ + "service.name": "datatracker", + "service.version": ietf.__version__, + "service.instance.id": worker.pid, + "service.namespace": "datatracker", + "deployment.environment.name": os.environ.get("DATATRACKER_SERVICE_ENV", "dev") + }) + trace.set_tracer_provider(TracerProvider(resource=resource)) + otlp_exporter = OTLPSpanExporter(endpoint="https://heimdall-otlp.ietf.org/v1/traces") + + trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(otlp_exporter)) + + # Instrumentations + if "all" in enabled_telemetry or "django" in enabled_telemetry: + DjangoInstrumentor().instrument() + if "all" in enabled_telemetry or "psycopg2" in enabled_telemetry: + Psycopg2Instrumentor().instrument() + if "all" in enabled_telemetry or "pymemcache" in enabled_telemetry: + PymemcacheInstrumentor().instrument() + if "all" in enabled_telemetry or "requests" in enabled_telemetry: + RequestsInstrumentor().instrument() diff --git a/dev/build/migration-start.sh b/dev/build/migration-start.sh index 901026e53b..578daf5cef 100644 --- a/dev/build/migration-start.sh +++ b/dev/build/migration-start.sh @@ -3,7 +3,11 @@ echo "Running Datatracker migrations..." ./ietf/manage.py migrate --settings=settings_local -echo "Running Blobdb migrations ..." -./ietf/manage.py migrate --settings=settings_local --database=blobdb +# Check whether the blobdb database exists - inspectdb will return a false +# status if not. +if ./ietf/manage.py inspectdb --database blobdb > /dev/null 2>&1; then + echo "Running Blobdb migrations ..." + ./ietf/manage.py migrate --settings=settings_local --database=blobdb +fi echo "Done!" diff --git a/ietf/secr/sreq/__init__.py b/dev/celery/Dockerfile similarity index 100% rename from ietf/secr/sreq/__init__.py rename to dev/celery/Dockerfile diff --git a/dev/deploy-to-container/settings_local.py b/dev/deploy-to-container/settings_local.py index aacf000093..055b48d0f5 100644 --- a/dev/deploy-to-container/settings_local.py +++ b/dev/deploy-to-container/settings_local.py @@ -71,11 +71,11 @@ DE_GFM_BINARY = '/usr/local/bin/de-gfm' -# No real secrets here, these are public testing values _only_ APP_API_TOKENS = { - "ietf.api.views.ingest_email_test": ["ingestion-test-token"] + "ietf.api.red_api" : ["devtoken", "redtoken"], # Not a real secret + "ietf.api.views.ingest_email_test": ["ingestion-test-token"], # Not a real secret + "ietf.api.views_rpc" : ["devtoken"], # Not a real secret } - # OIDC configuration SITE_URL = 'https://__HOSTNAME__' diff --git a/dev/k8s-get-deploy-name/.editorconfig b/dev/k8s-get-deploy-name/.editorconfig new file mode 100644 index 0000000000..fec5c66519 --- /dev/null +++ b/dev/k8s-get-deploy-name/.editorconfig @@ -0,0 +1,7 @@ +[*] +indent_size = 2 +indent_style = space +charset = utf-8 +trim_trailing_whitespace = false +end_of_line = lf +insert_final_newline = true diff --git a/dev/k8s-get-deploy-name/.gitignore b/dev/k8s-get-deploy-name/.gitignore new file mode 100644 index 0000000000..07e6e472cc --- /dev/null +++ b/dev/k8s-get-deploy-name/.gitignore @@ -0,0 +1 @@ +/node_modules diff --git a/dev/k8s-get-deploy-name/.npmrc b/dev/k8s-get-deploy-name/.npmrc new file mode 100644 index 0000000000..580a68c499 --- /dev/null +++ b/dev/k8s-get-deploy-name/.npmrc @@ -0,0 +1,3 @@ +audit = false +fund = false +save-exact = true diff --git a/dev/k8s-get-deploy-name/README.md b/dev/k8s-get-deploy-name/README.md new file mode 100644 index 0000000000..a6605e4dd2 --- /dev/null +++ b/dev/k8s-get-deploy-name/README.md @@ -0,0 +1,16 @@ +# Datatracker Get Deploy Name + +This tool process and slugify a git branch into an appropriate subdomain name. + +## Usage + +1. From the `dev/k8s-get-deploy-name` directory, install the dependencies: +```sh +npm install +``` +2. Run the command: (replacing the `branch` argument) +```sh +node /cli.js --branch feat/fooBar-123 +``` + +The subdomain name will be output. It can then be used in a workflow as a namespace name and subdomain value. diff --git a/dev/k8s-get-deploy-name/cli.js b/dev/k8s-get-deploy-name/cli.js new file mode 100644 index 0000000000..b6c3b5119e --- /dev/null +++ b/dev/k8s-get-deploy-name/cli.js @@ -0,0 +1,22 @@ +#!/usr/bin/env node + +import yargs from 'yargs/yargs' +import { hideBin } from 'yargs/helpers' +import slugify from 'slugify' + +const argv = yargs(hideBin(process.argv)).argv + +let branch = argv.branch +if (!branch) { + throw new Error('Missing --branch argument!') +} +if (branch.indexOf('/') >= 0) { + branch = branch.split('/').slice(1).join('-') +} +branch = slugify(branch, { lower: true, strict: true }) +if (branch.length < 1) { + throw new Error('Branch name is empty!') +} +process.stdout.write(`dt-${branch}`) + +process.exit(0) diff --git a/dev/k8s-get-deploy-name/package-lock.json b/dev/k8s-get-deploy-name/package-lock.json new file mode 100644 index 0000000000..e492a4cd38 --- /dev/null +++ b/dev/k8s-get-deploy-name/package-lock.json @@ -0,0 +1,303 @@ +{ + "name": "k8s-get-deploy-name", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "k8s-get-deploy-name", + "dependencies": { + "slugify": "1.6.6", + "yargs": "17.7.2" + } + }, + "node_modules/ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", + "engines": { + "node": ">=8" + } + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "node_modules/emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "node_modules/escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", + "engines": { + "node": ">=6" + } + }, + "node_modules/get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", + "engines": { + "node": "6.* || 8.* || >= 10.*" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", + "engines": { + "node": ">=8" + } + }, + "node_modules/require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/slugify": { + "version": "1.6.6", + "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.6.tgz", + "integrity": "sha512-h+z7HKHYXj6wJU+AnS/+IH8Uh9fdcX1Lrhg1/VMdf9PwoBQXFcXiAdsy2tSK0P6gKwJLXp02r90ahUCqHk9rrw==", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", + "engines": { + "node": ">=10" + } + }, + "node_modules/yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "dependencies": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", + "engines": { + "node": ">=12" + } + } + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" + }, + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "requires": { + "color-convert": "^2.0.1" + } + }, + "cliui": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", + "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", + "requires": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.1", + "wrap-ansi": "^7.0.0" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" + }, + "emoji-regex": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" + }, + "escalade": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", + "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==" + }, + "get-caller-file": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" + }, + "is-fullwidth-code-point": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" + }, + "require-directory": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==" + }, + "slugify": { + "version": "1.6.6", + "resolved": "https://registry.npmjs.org/slugify/-/slugify-1.6.6.tgz", + "integrity": "sha512-h+z7HKHYXj6wJU+AnS/+IH8Uh9fdcX1Lrhg1/VMdf9PwoBQXFcXiAdsy2tSK0P6gKwJLXp02r90ahUCqHk9rrw==" + }, + "string-width": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + } + }, + "strip-ansi": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "requires": { + "ansi-regex": "^5.0.1" + } + }, + "wrap-ansi": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "requires": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + } + }, + "y18n": { + "version": "5.0.8", + "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==" + }, + "yargs": { + "version": "17.7.2", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", + "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", + "requires": { + "cliui": "^8.0.1", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.3", + "y18n": "^5.0.5", + "yargs-parser": "^21.1.1" + } + }, + "yargs-parser": { + "version": "21.1.1", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==" + } + } +} diff --git a/dev/k8s-get-deploy-name/package.json b/dev/k8s-get-deploy-name/package.json new file mode 100644 index 0000000000..849f5d9b8d --- /dev/null +++ b/dev/k8s-get-deploy-name/package.json @@ -0,0 +1,8 @@ +{ + "name": "k8s-get-deploy-name", + "type": "module", + "dependencies": { + "slugify": "1.6.6", + "yargs": "17.7.2" + } +} diff --git a/docker-compose.yml b/docker-compose.yml index 100119c464..4c3f2f6b8e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,9 +13,10 @@ services: # network_mode: service:db depends_on: + - blobdb + - blobstore - db - mq - - blobstore ipc: host @@ -36,7 +37,7 @@ services: db: image: ghcr.io/ietf-tools/datatracker-db:latest # build: - # context: .. + # context: . # dockerfile: docker/db.Dockerfile restart: unless-stopped volumes: @@ -79,7 +80,10 @@ services: command: - '--loglevel=INFO' depends_on: + - blobdb + - blobstore - db + - mq restart: unless-stopped stop_grace_period: 1m volumes: @@ -102,7 +106,10 @@ services: - '--concurrency=1' depends_on: + - blobdb + - blobstore - db + - mq restart: unless-stopped stop_grace_period: 1m volumes: @@ -116,7 +123,7 @@ services: - "minio-data:/data" blobdb: - image: postgres:16 + image: postgres:17 restart: unless-stopped environment: POSTGRES_DB: blob @@ -129,7 +136,7 @@ services: # but can be enabled by uncommenting the following. # # beat: -# image: ghcr.io/ietf-tools/datatracker-celery:latest +# image: "${COMPOSE_PROJECT_NAME}-celery" # init: true # environment: # CELERY_APP: ietf diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile index fee3833733..dd4cf72ffd 100644 --- a/docker/app.Dockerfile +++ b/docker/app.Dockerfile @@ -10,12 +10,7 @@ ARG USER_GID=$USER_UID COPY docker/scripts/app-setup-debian.sh /tmp/library-scripts/docker-setup-debian.sh RUN sed -i 's/\r$//' /tmp/library-scripts/docker-setup-debian.sh && chmod +x /tmp/library-scripts/docker-setup-debian.sh -# Add Postgresql Apt Repository to get 14 -RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list -RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - - RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ - && apt-get install -y --no-install-recommends postgresql-client-14 pgloader \ # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131 && apt-get purge -y imagemagick imagemagick-6-common \ # Install common packages, non-root user diff --git a/docker/base.Dockerfile b/docker/base.Dockerfile index 57aac8ee56..2501636049 100644 --- a/docker/base.Dockerfile +++ b/docker/base.Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-bookworm +FROM python:3.12-bookworm LABEL maintainer="IETF Tools Team " ENV DEBIAN_FRONTEND=noninteractive @@ -11,21 +11,22 @@ RUN apt-get update \ # Add Node.js Source RUN apt-get install -y --no-install-recommends ca-certificates curl gnupg \ - && mkdir -p /etc/apt/keyrings\ - && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg -RUN echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list -RUN echo "Package: nodejs" >> /etc/apt/preferences.d/preferences && \ - echo "Pin: origin deb.nodesource.com" >> /etc/apt/preferences.d/preferences && \ - echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/preferences + && mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \ + && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list +RUN echo "Package: nodejs" >> /etc/apt/preferences.d/preferences \ + && echo "Pin: origin deb.nodesource.com" >> /etc/apt/preferences.d/preferences \ + && echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/preferences # Add Docker Source -RUN curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg -RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian \ - $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null - -# Add PostgreSQL Source -RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list -RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - +RUN mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker-archive-keyring.gpg \ + && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | tee /etc/apt/sources.list.d/docker.list + +# Add PostgreSQL Source +RUN mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /etc/apt/keyrings/apt.postgresql.org.gpg \ + && echo "deb [signed-by=/etc/apt/keyrings/apt.postgresql.org.gpg] https://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list # Install the packages we need RUN apt-get update --fix-missing && apt-get install -qy --no-install-recommends \ diff --git a/docker/celery.Dockerfile b/docker/celery.Dockerfile index e7c7b9cc3f..e93ca3cf77 100644 --- a/docker/celery.Dockerfile +++ b/docker/celery.Dockerfile @@ -10,12 +10,7 @@ ARG USER_GID=$USER_UID COPY docker/scripts/app-setup-debian.sh /tmp/library-scripts/docker-setup-debian.sh RUN sed -i 's/\r$//' /tmp/library-scripts/docker-setup-debian.sh && chmod +x /tmp/library-scripts/docker-setup-debian.sh -# Add Postgresql Apt Repository to get 14 -RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(. /etc/os-release && echo "$VERSION_CODENAME")-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list -RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - - RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ - && apt-get install -y --no-install-recommends postgresql-client-14 pgloader \ # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131 && apt-get purge -y imagemagick imagemagick-6-common \ # Install common packages, non-root user diff --git a/docker/configs/nginx-proxy.conf b/docker/configs/nginx-proxy.conf index 3068cc71d7..5a9ae31ad0 100644 --- a/docker/configs/nginx-proxy.conf +++ b/docker/configs/nginx-proxy.conf @@ -4,6 +4,7 @@ server { proxy_read_timeout 1d; proxy_send_timeout 1d; + client_max_body_size 0; # disable checking root /var/www/html; index index.html index.htm index.nginx-debian.html; diff --git a/docker/configs/settings_local.py b/docker/configs/settings_local.py index ca51871463..94adc516a4 100644 --- a/docker/configs/settings_local.py +++ b/docker/configs/settings_local.py @@ -89,6 +89,8 @@ secret_key="minio_pass", security_token=None, client_config=botocore.config.Config( + request_checksum_calculation="when_required", + response_checksum_validation="when_required", signature_version="s3v4", connect_timeout=BLOBSTORAGE_CONNECT_TIMEOUT, read_timeout=BLOBSTORAGE_READ_TIMEOUT, @@ -98,3 +100,23 @@ bucket_name=f"{storagename}", ), } + +# For dev on rfc-index generation, create a red_bucket/ directory in the project root +# and uncomment these settings. Generated files will appear in this directory. To +# generate an accurate index, put up-to-date copies of unusable-rfc-numbers.json, +# april-first-rfc-numbers.json, and publication-std-levels.json in this directory +# before generating the index. +# +# STORAGES["red_bucket"] = { +# "BACKEND": "django.core.files.storage.FileSystemStorage", +# "OPTIONS": {"location": "red_bucket"}, +# } + +APP_API_TOKENS = { + "ietf.api.red_api" : ["devtoken", "redtoken"], # Not a real secret + "ietf.api.views_rpc" : ["devtoken"], # Not a real secret +} + +# Errata system api configuration +ERRATA_METADATA_NOTIFICATION_URL = "http://host.docker.internal:8808/api/rfc_metadata_update/" +ERRATA_METADATA_NOTIFICATION_API_KEY = "not a real secret" diff --git a/docker/configs/settings_local_vite.py b/docker/configs/settings_local_vite.py index 7fb12a003d..9116905b12 100644 --- a/docker/configs/settings_local_vite.py +++ b/docker/configs/settings_local_vite.py @@ -2,5 +2,9 @@ # -*- coding: utf-8 -*- from ietf.settings_local import * # pyflakes:ignore +from ietf.settings_local import DJANGO_VITE -DJANGO_VITE_DEV_MODE = True +DJANGO_VITE["default"] |= { + "dev_mode": True, + "dev_server_port": 3000, +} diff --git a/docker/docker-compose.extend.yml b/docker/docker-compose.extend.yml index a69a453110..12ebe447d5 100644 --- a/docker/docker-compose.extend.yml +++ b/docker/docker-compose.extend.yml @@ -18,8 +18,8 @@ services: - '5433' blobstore: ports: - - '9000' - - '9001' + - '9000:9000' + - '9001:9001' celery: volumes: - .:/workspace diff --git a/docker/scripts/app-configure-blobstore.py b/docker/scripts/app-configure-blobstore.py index df4685b246..3140e39306 100755 --- a/docker/scripts/app-configure-blobstore.py +++ b/docker/scripts/app-configure-blobstore.py @@ -17,7 +17,11 @@ def init_blobstore(): aws_access_key_id=os.environ.get("BLOB_STORE_ACCESS_KEY", "minio_root"), aws_secret_access_key=os.environ.get("BLOB_STORE_SECRET_KEY", "minio_pass"), aws_session_token=None, - config=botocore.config.Config(signature_version="s3v4"), + config=botocore.config.Config( + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + signature_version="s3v4", + ), ) for bucketname in ARTIFACT_STORAGE_NAMES: try: diff --git a/docker/scripts/app-init.sh b/docker/scripts/app-init.sh index 17e0c6c764..1d895cdf53 100755 --- a/docker/scripts/app-init.sh +++ b/docker/scripts/app-init.sh @@ -108,7 +108,7 @@ echo "Running initial checks..." if [ -z "$EDITOR_VSCODE" ]; then CODE=0 - python -m smtpd -n -c DebuggingServer localhost:2025 & + python -m aiosmtpd -n -c ietf.utils.aiosmtpd.DevDebuggingHandler -l localhost:2025 & if [ -z "$*" ]; then echo "-----------------------------------------------------------------" echo "Ready!" diff --git a/ietf/api/__init__.py b/ietf/api/__init__.py index 230f8339bd..d4562f97dd 100644 --- a/ietf/api/__init__.py +++ b/ietf/api/__init__.py @@ -181,7 +181,7 @@ class Serializer(tastypie.serializers.Serializer): OPTION_ESCAPE_NULLS = "datatracker-escape-nulls" def format_datetime(self, data): - return data.astimezone(datetime.timezone.utc).replace(tzinfo=None).isoformat(timespec="seconds") + "Z" + return data.astimezone(datetime.UTC).replace(tzinfo=None).isoformat(timespec="seconds") + "Z" def to_simple(self, data, options): options = options or {} diff --git a/ietf/api/routers.py b/ietf/api/routers.py index 745ddaa811..99afdb242a 100644 --- a/ietf/api/routers.py +++ b/ietf/api/routers.py @@ -3,14 +3,29 @@ from django.core.exceptions import ImproperlyConfigured from rest_framework import routers -class PrefixedSimpleRouter(routers.SimpleRouter): - """SimpleRouter that adds a dot-separated prefix to its basename""" + +class PrefixedBasenameMixin: + """Mixin to add a prefix to the basename of a rest_framework BaseRouter""" def __init__(self, name_prefix="", *args, **kwargs): self.name_prefix = name_prefix if len(self.name_prefix) == 0 or self.name_prefix[-1] == ".": raise ImproperlyConfigured("Cannot use a name_prefix that is empty or ends with '.'") super().__init__(*args, **kwargs) - def get_default_basename(self, viewset): - basename = super().get_default_basename(viewset) - return f"{self.name_prefix}.{basename}" + def register(self, prefix, viewset, basename=None): + # Get the superclass "register" method from the class this is mixed-in with. + # This avoids typing issues with calling super().register() directly in a + # mixin class. + super_register = getattr(super(), "register") + if not super_register or not callable(super_register): + raise TypeError("Must mixin with superclass that has register() method") + super_register(prefix, viewset, basename=f"{self.name_prefix}.{basename}") + + +class PrefixedSimpleRouter(PrefixedBasenameMixin, routers.SimpleRouter): + """SimpleRouter that adds a dot-separated prefix to its basename""" + + +class PrefixedDefaultRouter(PrefixedBasenameMixin, routers.DefaultRouter): + """DefaultRouter that adds a dot-separated prefix to its basename""" + diff --git a/ietf/api/serializers_rpc.py b/ietf/api/serializers_rpc.py new file mode 100644 index 0000000000..c17cbc64ce --- /dev/null +++ b/ietf/api/serializers_rpc.py @@ -0,0 +1,764 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +import datetime +from pathlib import Path +from typing import Literal, Optional + +from django.db import transaction +from django.urls import reverse as urlreverse +from django.utils import timezone +from drf_spectacular.types import OpenApiTypes +from drf_spectacular.utils import extend_schema_field +from rest_framework import serializers + +from ietf.doc.expire import move_draft_files_to_archive +from ietf.doc.models import ( + DocumentAuthor, + Document, + RelatedDocument, + State, + DocEvent, + RfcAuthor, +) +from ietf.doc.serializers import RfcAuthorSerializer +from ietf.doc.utils import ( + default_consensus, + prettify_std_name, + update_action_holders, + update_rfcauthors, +) +from ietf.group.models import Group +from ietf.group.serializers import AreaSerializer +from ietf.name.models import StreamName, StdLevelName +from ietf.person.models import Person +from ietf.utils import log + + +class PersonSerializer(serializers.ModelSerializer): + email = serializers.EmailField(read_only=True) + picture = serializers.URLField(source="cdn_photo_url", read_only=True) + url = serializers.SerializerMethodField( + help_text="relative URL for datatracker person page" + ) + + class Meta: + model = Person + fields = ["id", "plain_name", "email", "picture", "url"] + read_only_fields = ["id", "plain_name", "email", "picture", "url"] + + @extend_schema_field(OpenApiTypes.URI) + def get_url(self, object: Person): + return urlreverse( + "ietf.person.views.profile", + kwargs={"email_or_name": object.email_address() or object.name}, + ) + + +class EmailPersonSerializer(serializers.Serializer): + email = serializers.EmailField(source="address") + person_pk = serializers.IntegerField(source="person.pk") + name = serializers.CharField(source="person.name") + last_name = serializers.CharField(source="person.last_name") + initials = serializers.CharField(source="person.initials") + + +class LowerCaseEmailField(serializers.EmailField): + def to_representation(self, value): + return super().to_representation(value).lower() + + +class AuthorPersonSerializer(serializers.ModelSerializer): + person_pk = serializers.IntegerField(source="pk", read_only=True) + last_name = serializers.CharField() + initials = serializers.CharField() + email_addresses = serializers.ListField( + source="email_set.all", child=LowerCaseEmailField() + ) + + class Meta: + model = Person + fields = ["person_pk", "name", "last_name", "initials", "email_addresses"] + + +class RfcWithAuthorsSerializer(serializers.ModelSerializer): + authors = AuthorPersonSerializer(many=True, source="author_persons") + + class Meta: + model = Document + fields = ["rfc_number", "authors"] + + +class DraftWithAuthorsSerializer(serializers.ModelSerializer): + draft_name = serializers.CharField(source="name") + authors = AuthorPersonSerializer(many=True, source="author_persons") + + class Meta: + model = Document + fields = ["draft_name", "authors"] + + +class DocumentAuthorSerializer(serializers.ModelSerializer): + """Serializer for a Person in a response""" + + plain_name = serializers.SerializerMethodField() + + class Meta: + model = DocumentAuthor + fields = ["person", "plain_name"] + + def get_plain_name(self, document_author: DocumentAuthor) -> str: + return document_author.person.plain_name() + + +class FullDraftSerializer(serializers.ModelSerializer): + # Redefine these fields so they don't pick up the regex validator patterns. + # There seem to be some non-compliant drafts in the system! If this serializer + # is used for a writeable view, the validation will need to be added back. + name = serializers.CharField(max_length=255) + title = serializers.CharField(max_length=255) + group = serializers.SlugRelatedField(slug_field="acronym", read_only=True) + area = AreaSerializer(read_only=True) + + # Other fields we need to add / adjust + source_format = serializers.SerializerMethodField() + authors = DocumentAuthorSerializer(many=True, source="documentauthor_set") + shepherd = serializers.PrimaryKeyRelatedField( + source="shepherd.person", read_only=True + ) + consensus = serializers.SerializerMethodField() + + class Meta: + model = Document + fields = [ + "id", + "name", + "rev", + "stream", + "title", + "group", + "area", + "abstract", + "pages", + "source_format", + "authors", + "intended_std_level", + "consensus", + "shepherd", + "ad", + ] + + def get_consensus(self, doc: Document) -> Optional[bool]: + return default_consensus(doc) + + def get_source_format( + self, doc: Document + ) -> Literal["unknown", "xml-v2", "xml-v3", "txt"]: + submission = doc.submission() + if submission is None: + return "unknown" + if ".xml" in submission.file_types: + if submission.xml_version == "3": + return "xml-v3" + else: + return "xml-v2" + elif ".txt" in submission.file_types: + return "txt" + return "unknown" + + +class DraftSerializer(FullDraftSerializer): + class Meta: + model = Document + fields = [ + "id", + "name", + "rev", + "stream", + "title", + "group", + "pages", + "source_format", + "authors", + "consensus", + ] + + +class SubmittedToQueueSerializer(FullDraftSerializer): + submitted = serializers.SerializerMethodField() + consensus = serializers.SerializerMethodField() + + class Meta: + model = Document + fields = [ + "id", + "name", + "stream", + "submitted", + "consensus", + ] + + def get_submitted(self, doc) -> Optional[datetime.datetime]: + event = doc.sent_to_rfc_editor_event() + return None if event is None else event.time + + def get_consensus(self, doc) -> Optional[bool]: + return default_consensus(doc) + + +class OriginalStreamSerializer(serializers.ModelSerializer): + stream = serializers.CharField(read_only=True, source="orig_stream_id") + + class Meta: + model = Document + fields = ["rfc_number", "stream"] + + +class ReferenceSerializer(serializers.ModelSerializer): + class Meta: + model = Document + fields = ["id", "name"] + read_only_fields = ["id", "name"] + + +def _update_authors(rfc, authors_data): + # Construct unsaved instances from validated author data + new_authors = [RfcAuthor(**authdata) for authdata in authors_data] + # Update the RFC with the new author set + with transaction.atomic(): + change_events = update_rfcauthors(rfc, new_authors) + for event in change_events: + event.save() + return change_events + + +class SubseriesNameField(serializers.RegexField): + + def __init__(self, **kwargs): + # pattern: no leading 0, finite length (arbitrarily set to 5 digits) + regex = r"^(bcp|std|fyi)[1-9][0-9]{0,4}$" + super().__init__(regex, **kwargs) + + + +class RfcPubSerializer(serializers.ModelSerializer): + """Write-only serializer for RFC publication""" + # publication-related fields + published = serializers.DateTimeField(default_timezone=datetime.timezone.utc) + draft_name = serializers.RegexField( + required=False, regex=r"^draft-[a-zA-Z0-9-]+$" + ) + draft_rev = serializers.RegexField( + required=False, regex=r"^[0-9][0-9]$" + ) + + # fields on the RFC Document that need tweaking from ModelSerializer defaults + rfc_number = serializers.IntegerField(min_value=1, required=True) + group = serializers.SlugRelatedField( + slug_field="acronym", queryset=Group.objects.all(), required=False + ) + stream = serializers.PrimaryKeyRelatedField( + queryset=StreamName.objects.filter(used=True) + ) + std_level = serializers.PrimaryKeyRelatedField( + queryset=StdLevelName.objects.filter(used=True), + ) + ad = serializers.PrimaryKeyRelatedField( + queryset=Person.objects.all(), + allow_null=True, + required=False, + ) + obsoletes = serializers.SlugRelatedField( + many=True, + required=False, + slug_field="rfc_number", + queryset=Document.objects.filter(type_id="rfc"), + ) + updates = serializers.SlugRelatedField( + many=True, + required=False, + slug_field="rfc_number", + queryset=Document.objects.filter(type_id="rfc"), + ) + subseries = serializers.ListField(child=SubseriesNameField(required=False)) + # N.b., authors is _not_ a field on Document! + authors = RfcAuthorSerializer(many=True) + + class Meta: + model = Document + fields = [ + "published", + "draft_name", + "draft_rev", + "rfc_number", + "title", + "authors", + "group", + "stream", + "abstract", + "pages", + "std_level", + "ad", + "obsoletes", + "updates", + "subseries", + "keywords", + ] + + def validate(self, data): + if "draft_name" in data or "draft_rev" in data: + if "draft_name" not in data: + raise serializers.ValidationError( + {"draft_name": "Missing draft_name"}, + code="invalid-draft-spec", + ) + if "draft_rev" not in data: + raise serializers.ValidationError( + {"draft_rev": "Missing draft_rev"}, + code="invalid-draft-spec", + ) + return data + + def update(self, instance, validated_data): + raise RuntimeError("Cannot update with this serializer") + + def create(self, validated_data): + """Publish an RFC""" + published = validated_data.pop("published") + draft_name = validated_data.pop("draft_name", None) + draft_rev = validated_data.pop("draft_rev", None) + obsoletes = validated_data.pop("obsoletes", []) + updates = validated_data.pop("updates", []) + subseries = validated_data.pop("subseries", []) + + system_person = Person.objects.get(name="(System)") + + # If specified, retrieve draft and extract RFC default values from it + if draft_name is None: + draft = None + else: + # validation enforces that draft_name and draft_rev are both present + draft = Document.objects.filter( + type_id="draft", + name=draft_name, + rev=draft_rev, + ).first() + if draft is None: + raise serializers.ValidationError( + { + "draft_name": "No such draft", + "draft_rev": "No such draft", + }, + code="invalid-draft" + ) + elif draft.get_state_slug() == "rfc": + raise serializers.ValidationError( + { + "draft_name": "Draft already published as RFC", + }, + code="already-published-draft", + ) + + # Transaction to clean up if something fails + with transaction.atomic(): + # create rfc, letting validated request data override draft defaults + rfc = self._create_rfc(validated_data) + DocEvent.objects.create( + doc=rfc, + rev=rfc.rev, + type="published_rfc", + time=published, + by=system_person, + desc="RFC published", + ) + rfc.set_state(State.objects.get(used=True, type_id="rfc", slug="published")) + + # create updates / obsoletes relations + for obsoleted_rfc_pk in obsoletes: + RelatedDocument.objects.get_or_create( + source=rfc, target=obsoleted_rfc_pk, relationship_id="obs" + ) + for updated_rfc_pk in updates: + RelatedDocument.objects.get_or_create( + source=rfc, target=updated_rfc_pk, relationship_id="updates" + ) + + # create subseries relations + for subseries_doc_name in subseries: + ss_slug = subseries_doc_name[:3] + subseries_doc, ss_doc_created = Document.objects.get_or_create( + type_id=ss_slug, name=subseries_doc_name + ) + if ss_doc_created: + subseries_doc.docevent_set.create( + type=f"{ss_slug}_doc_created", + by=system_person, + desc=f"Created {subseries_doc_name} via publication of {rfc.name}", + ) + _, ss_rel_created = subseries_doc.relateddocument_set.get_or_create( + relationship_id="contains", target=rfc + ) + if ss_rel_created: + subseries_doc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Added {rfc.name} to {subseries_doc.name}", + ) + rfc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Added {rfc.name} to {subseries_doc.name}", + ) + + + # create relation with draft and update draft state + if draft is not None: + draft_changes = [] + draft_events = [] + if draft.get_state_slug() != "rfc": + draft.set_state( + State.objects.get(used=True, type="draft", slug="rfc") + ) + move_draft_files_to_archive(draft, draft.rev) + draft_changes.append(f"changed state to {draft.get_state()}") + + r, created_relateddoc = RelatedDocument.objects.get_or_create( + source=draft, target=rfc, relationship_id="became_rfc", + ) + if created_relateddoc: + change = "created {rel_name} relationship between {pretty_draft_name} and {pretty_rfc_name}".format( + rel_name=r.relationship.name.lower(), + pretty_draft_name=prettify_std_name(draft_name), + pretty_rfc_name=prettify_std_name(rfc.name), + ) + draft_changes.append(change) + + # Always set the "draft-iesg" state. This state should be set for all drafts, so + # log a warning if it is not set. What should happen here is that ietf stream + # RFCs come in as "rfcqueue" and are set to "pub" when they appear in the RFC index. + # Other stream documents should normally be "idexists" and be left that way. The + # code here *actually* leaves "draft-iesg" state alone if it is "idexists" or "pub", + # and changes any other state to "pub". If unset, it changes it to "idexists". + # This reflects historical behavior and should probably be updated, but a migration + # of existing drafts (and validation of the change) is needed before we change the + # handling. + prev_iesg_state = draft.get_state("draft-iesg") + if prev_iesg_state is None: + log.log(f'Warning while processing {rfc.name}: {draft.name} has no "draft-iesg" state') + new_iesg_state = State.objects.get(type_id="draft-iesg", slug="idexists") + elif prev_iesg_state.slug not in ("pub", "idexists"): + if prev_iesg_state.slug != "rfcqueue": + log.log( + 'Warning while processing {}: {} is in "draft-iesg" state {} (expected "rfcqueue")'.format( + rfc.name, draft.name, prev_iesg_state.slug + ) + ) + new_iesg_state = State.objects.get(type_id="draft-iesg", slug="pub") + else: + new_iesg_state = prev_iesg_state + + if new_iesg_state != prev_iesg_state: + draft.set_state(new_iesg_state) + draft_changes.append(f"changed {new_iesg_state.type.label} to {new_iesg_state}") + e = update_action_holders(draft, prev_iesg_state, new_iesg_state) + if e: + draft_events.append(e) + + # If the draft and RFC streams agree, move draft to "pub" stream state. If not, complain. + if draft.stream != rfc.stream: + log.log("Warning while processing {}: draft {} stream is {} but RFC stream is {}".format( + rfc.name, draft.name, draft.stream, rfc.stream + )) + elif draft.stream.slug in ["iab", "irtf", "ise", "editorial"]: + stream_slug = f"draft-stream-{draft.stream.slug}" + prev_state = draft.get_state(stream_slug) + if prev_state is not None and prev_state.slug != "pub": + new_state = State.objects.select_related("type").get(used=True, type__slug=stream_slug, slug="pub") + draft.set_state(new_state) + draft_changes.append( + f"changed {new_state.type.label} to {new_state}" + ) + e = update_action_holders(draft, prev_state, new_state) + if e: + draft_events.append(e) + if draft_changes: + draft_events.append( + DocEvent.objects.create( + doc=draft, + rev=draft.rev, + by=system_person, + type="sync_from_rfc_editor", + desc=f"Updated while publishing {rfc.name} ({', '.join(draft_changes)})", + ) + ) + draft.save_with_history(draft_events) + + return rfc + + def _create_rfc(self, validated_data): + authors_data = validated_data.pop("authors") + rfc = Document.objects.create( + type_id="rfc", + name=f"rfc{validated_data['rfc_number']}", + **validated_data, + ) + for order, author_data in enumerate(authors_data): + rfc.rfcauthor_set.create( + order=order, + **author_data, + ) + return rfc + + +class EditableRfcSerializer(serializers.ModelSerializer): + # Would be nice to reconcile this with ietf.doc.serializers.RfcSerializer. + # The purposes of that serializer (representing data for Red) and this one + # (accepting updates from Purple) are different enough that separate formats + # may be needed, but if not it'd be nice to have a single RfcSerializer that + # can serve both. + # + # Should also consider whether this and RfcPubSerializer should merge. + # + # Treats published and subseries fields as write-only. This isn't quite correct, + # but makes it easier and we don't currently use the serialized value except for + # debugging. + published = serializers.DateTimeField( + default_timezone=datetime.timezone.utc, + write_only=True, + ) + authors = RfcAuthorSerializer(many=True, min_length=1, source="rfcauthor_set") + subseries = serializers.ListField( + child=SubseriesNameField(required=False), + write_only=True, + ) + + class Meta: + model = Document + fields = [ + "published", + "title", + "authors", + "stream", + "abstract", + "pages", + "std_level", + "subseries", + "keywords", + ] + + def create(self, validated_data): + raise RuntimeError("Cannot create with this serializer") + + def update(self, instance, validated_data): + assert isinstance(instance, Document) + assert instance.type_id == "rfc" + rfc = instance # get better name + + system_person = Person.objects.get(name="(System)") + + # Remove data that needs special handling. Use a singleton object to detect + # missing values in case we ever support a value that needs None as an option. + omitted = object() + published = validated_data.pop("published", omitted) + subseries = validated_data.pop("subseries", omitted) + authors_data = validated_data.pop("rfcauthor_set", omitted) + + # Transaction to clean up if something fails + with transaction.atomic(): + # update the rfc Document itself + rfc_changes = [] + rfc_events = [] + + for attr, new_value in validated_data.items(): + old_value = getattr(rfc, attr) + if new_value != old_value: + rfc_changes.append( + f"changed {attr} to '{new_value}' from '{old_value}'" + ) + setattr(rfc, attr, new_value) + if len(rfc_changes) > 0: + rfc_change_summary = f"{', '.join(rfc_changes)}" + rfc_events.append( + DocEvent.objects.create( + doc=rfc, + rev=rfc.rev, + by=system_person, + type="sync_from_rfc_editor", + desc=f"Changed metadata: {rfc_change_summary}", + ) + ) + if authors_data is not omitted: + rfc_events.extend(_update_authors(instance, authors_data)) + + if published is not omitted: + published_event = rfc.latest_event(type="published_rfc") + if published_event is None: + # unexpected, but possible in theory + rfc_events.append( + DocEvent.objects.create( + doc=rfc, + rev=rfc.rev, + type="published_rfc", + time=published, + by=system_person, + desc="RFC published", + ) + ) + rfc_events.append( + DocEvent.objects.create( + doc=rfc, + rev=rfc.rev, + type="sync_from_rfc_editor", + by=system_person, + desc=( + f"Set publication timestamp to {published.isoformat()}" + ), + ) + ) + else: + original_pub_time = published_event.time + if published != original_pub_time: + published_event.time = published + published_event.save() + rfc_events.append( + DocEvent.objects.create( + doc=rfc, + rev=rfc.rev, + type="sync_from_rfc_editor", + by=system_person, + desc=( + f"Changed publication time to " + f"{published.isoformat()} from " + f"{original_pub_time.isoformat()}" + ) + ) + ) + + # update subseries relations + if subseries is not omitted: + for subseries_doc_name in subseries: + ss_slug = subseries_doc_name[:3] + subseries_doc, ss_doc_created = Document.objects.get_or_create( + type_id=ss_slug, name=subseries_doc_name + ) + if ss_doc_created: + subseries_doc.docevent_set.create( + type=f"{ss_slug}_doc_created", + by=system_person, + desc=f"Created {subseries_doc_name} via update of {rfc.name}", + ) + _, ss_rel_created = subseries_doc.relateddocument_set.get_or_create( + relationship_id="contains", target=rfc + ) + if ss_rel_created: + subseries_doc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Added {rfc.name} to {subseries_doc.name}", + ) + rfc_events.append( + rfc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Added {rfc.name} to {subseries_doc.name}", + ) + ) + # Delete subseries relations that are no longer current + stale_subseries_relations = rfc.relations_that("contains").exclude( + source__name__in=subseries + ) + for stale_relation in stale_subseries_relations: + stale_subseries_doc = stale_relation.source + rfc_events.append( + rfc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Removed {rfc.name} from {stale_subseries_doc.name}", + ) + ) + stale_subseries_doc.docevent_set.create( + type="sync_from_rfc_editor", + by=system_person, + desc=f"Removed {rfc.name} from {stale_subseries_doc.name}", + ) + stale_subseries_relations.delete() + if len(rfc_events) > 0: + rfc.save_with_history(rfc_events) + return rfc + + +class RfcFileSerializer(serializers.Serializer): + # The structure of this serializer is constrained by what openapi-generator-cli's + # python generator can correctly serialize as multipart/form-data. It does not + # handle nested serializers well (or perhaps at all). ListFields with child + # ChoiceField or RegexField do not serialize correctly. DictFields don't seem + # to work. + # + # It does seem to correctly send filenames along with FileFields, even as a child + # in a ListField, so we use that to convey the file format of each item. There + # are other options we could consider (e.g., a structured CharField) but this + # works. + allowed_extensions = ( + ".html", + ".json", + ".notprepped.xml", + ".pdf", + ".txt", + ".xml", + ) + + rfc = serializers.SlugRelatedField( + slug_field="rfc_number", + queryset=Document.objects.filter(type_id="rfc"), + help_text="RFC number to which the contents belong", + ) + contents = serializers.ListField( + child=serializers.FileField( + allow_empty_file=False, + use_url=False, + ), + help_text=( + "List of content files. Filename extensions are used to identify " + "file types, but filenames are otherwise ignored." + ), + ) + mtime = serializers.DateTimeField( + required=False, + default=timezone.now, + default_timezone=datetime.UTC, + help_text="Modification timestamp to apply to uploaded files", + ) + replace = serializers.BooleanField( + required=False, + default=False, + help_text=( + "Replace existing files for this RFC. Defaults to false. When false, " + "if _any_ files already exist for the specified RFC the upload will be " + "rejected regardless of which files are being uploaded. When true," + "existing files will be removed and new ones will be put in place. BE" + "VERY CAREFUL WITH THIS OPTION IN PRODUCTION." + ), + ) + + def validate_contents(self, data): + found_extensions = [] + for uploaded_file in data: + if not hasattr(uploaded_file, "name"): + raise serializers.ValidationError( + "filename not specified for uploaded file", + code="missing-filename", + ) + ext = "".join(Path(uploaded_file.name).suffixes) + if ext not in self.allowed_extensions: + raise serializers.ValidationError( + f"File uploaded with invalid extension '{ext}'", + code="invalid-filename-ext", + ) + if ext in found_extensions: + raise serializers.ValidationError( + f"More than one file uploaded with extension '{ext}'", + code="duplicate-filename-ext", + ) + return data + + +class NotificationAckSerializer(serializers.Serializer): + message = serializers.CharField(default="ack") diff --git a/ietf/api/tests.py b/ietf/api/tests.py index 865f877bfb..2a44791a5c 100644 --- a/ietf/api/tests.py +++ b/ietf/api/tests.py @@ -462,12 +462,12 @@ def test_api_add_session_attendees(self): self.assertTrue(session.attended_set.filter(person=recman).exists()) self.assertEqual( session.attended_set.get(person=recman).time, - datetime.datetime(2023, 9, 3, 12, 34, 56, tzinfo=datetime.timezone.utc), + datetime.datetime(2023, 9, 3, 12, 34, 56, tzinfo=datetime.UTC), ) self.assertTrue(session.attended_set.filter(person=otherperson).exists()) self.assertEqual( session.attended_set.get(person=otherperson).time, - datetime.datetime(2023, 9, 3, 3, 0, 19, tzinfo=datetime.timezone.utc), + datetime.datetime(2023, 9, 3, 3, 0, 19, tzinfo=datetime.UTC), ) def test_api_upload_polls_and_chatlog(self): @@ -871,7 +871,7 @@ def test_api_new_meeting_registration_v2_nomcom(self): self.assertEqual(volunteer.origin, 'registration') def test_api_version(self): - DumpInfo.objects.create(date=timezone.datetime(2022,8,31,7,10,1,tzinfo=datetime.timezone.utc), host='testapi.example.com',tz='UTC') + DumpInfo.objects.create(date=timezone.datetime(2022,8,31,7,10,1,tzinfo=datetime.UTC), host='testapi.example.com',tz='UTC') url = urlreverse('ietf.api.views.version') r = self.client.get(url) data = r.json() diff --git a/ietf/api/tests_serializers_rpc.py b/ietf/api/tests_serializers_rpc.py new file mode 100644 index 0000000000..1babb4c30f --- /dev/null +++ b/ietf/api/tests_serializers_rpc.py @@ -0,0 +1,139 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +from django.utils import timezone + +from ietf.utils.test_utils import TestCase +from ietf.doc.models import Document +from ietf.doc.factories import WgRfcFactory +from .serializers_rpc import EditableRfcSerializer + + +class EditableRfcSerializerTests(TestCase): + def test_create(self): + serializer = EditableRfcSerializer( + data={ + "published": timezone.now(), + "title": "Yadda yadda yadda", + "authors": [ + { + "titlepage_name": "B. Fett", + "is_editor": False, + "affiliation": "DBA Galactic Empire", + "country": "", + }, + ], + "stream": "ietf", + "abstract": "A long time ago in a galaxy far, far away...", + "pages": 3, + "std_level": "inf", + "subseries": ["fyi999"], + } + ) + self.assertTrue(serializer.is_valid()) + with self.assertRaises(RuntimeError, msg="serializer does not allow create()"): + serializer.save() + + def test_update(self): + rfc = WgRfcFactory(pages=10) + serializer = EditableRfcSerializer( + instance=rfc, + data={ + "published": timezone.now(), + "title": "Yadda yadda yadda", + "authors": [ + { + "titlepage_name": "B. Fett", + "is_editor": False, + "affiliation": "DBA Galactic Empire", + "country": "", + }, + ], + "stream": "ise", + "abstract": "A long time ago in a galaxy far, far away...", + "pages": 3, + "std_level": "inf", + "subseries": ["fyi999"], + }, + ) + self.assertTrue(serializer.is_valid()) + result = serializer.save() + result.refresh_from_db() + self.assertEqual(result.title, "Yadda yadda yadda") + self.assertEqual( + list( + result.rfcauthor_set.values( + "titlepage_name", "is_editor", "affiliation", "country" + ) + ), + [ + { + "titlepage_name": "B. Fett", + "is_editor": False, + "affiliation": "DBA Galactic Empire", + "country": "", + }, + ], + ) + self.assertEqual(result.stream_id, "ise") + self.assertEqual( + result.abstract, "A long time ago in a galaxy far, far away..." + ) + self.assertEqual(result.pages, 3) + self.assertEqual(result.std_level_id, "inf") + self.assertEqual( + result.part_of(), + [Document.objects.get(name="fyi999")], + ) + + def test_partial_update(self): + # We could test other permutations of fields, but authors is a partial update + # we know we are going to use, so verifying that one in particular. + rfc = WgRfcFactory(pages=10, abstract="do or do not", title="padawan") + serializer = EditableRfcSerializer( + partial=True, + instance=rfc, + data={ + "authors": [ + { + "titlepage_name": "B. Fett", + "is_editor": False, + "affiliation": "DBA Galactic Empire", + "country": "", + }, + ], + }, + ) + self.assertTrue(serializer.is_valid()) + result = serializer.save() + result.refresh_from_db() + self.assertEqual(rfc.title, "padawan") + self.assertEqual( + list( + result.rfcauthor_set.values( + "titlepage_name", "is_editor", "affiliation", "country" + ) + ), + [ + { + "titlepage_name": "B. Fett", + "is_editor": False, + "affiliation": "DBA Galactic Empire", + "country": "", + }, + ], + ) + self.assertEqual(result.stream_id, "ietf") + self.assertEqual(result.abstract, "do or do not") + self.assertEqual(result.pages, 10) + self.assertEqual(result.std_level_id, "ps") + self.assertEqual(result.part_of(), []) + + # Test only a field on the Document itself to be sure that it works + serializer = EditableRfcSerializer( + partial=True, + instance=rfc, + data={"title": "jedi master"}, + ) + self.assertTrue(serializer.is_valid()) + result = serializer.save() + result.refresh_from_db() + self.assertEqual(rfc.title, "jedi master") diff --git a/ietf/api/tests_views_rpc.py b/ietf/api/tests_views_rpc.py new file mode 100644 index 0000000000..7ab8778d28 --- /dev/null +++ b/ietf/api/tests_views_rpc.py @@ -0,0 +1,380 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +from io import StringIO +from pathlib import Path +from tempfile import TemporaryDirectory + +from django.conf import settings +from django.core.files.base import ContentFile +from django.db.models import Max +from django.db.models.functions import Coalesce +from django.test.utils import override_settings +from django.urls import reverse as urlreverse +import mock + +from ietf.blobdb.models import Blob +from ietf.doc.factories import IndividualDraftFactory, RfcFactory, WgDraftFactory, WgRfcFactory +from ietf.doc.models import RelatedDocument, Document +from ietf.group.factories import RoleFactory, GroupFactory +from ietf.person.factories import PersonFactory +from ietf.utils.test_utils import APITestCase, reload_db_objects + + +class RpcApiTests(APITestCase): + @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + def test_draftviewset_references(self): + viewname = "ietf.api.purple_api.draft-references" + + # non-existent draft + bad_id = Document.objects.aggregate(unused_id=Coalesce(Max("id"), 0) + 100)[ + "unused_id" + ] + url = urlreverse(viewname, kwargs={"doc_id": bad_id}) + # Without credentials + r = self.client.get(url) + self.assertEqual(r.status_code, 403) + # Add credentials + r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 404) + + # draft without any normative references + draft = IndividualDraftFactory() + draft = reload_db_objects(draft) + url = urlreverse(viewname, kwargs={"doc_id": draft.id}) + r = self.client.get(url) + self.assertEqual(r.status_code, 403) + r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 200) + refs = r.json() + self.assertEqual(refs, []) + + # draft without any normative references but with an informative reference + draft_foo = IndividualDraftFactory() + draft_foo = reload_db_objects(draft_foo) + RelatedDocument.objects.create( + source=draft, target=draft_foo, relationship_id="refinfo" + ) + url = urlreverse(viewname, kwargs={"doc_id": draft.id}) + r = self.client.get(url) + self.assertEqual(r.status_code, 403) + r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 200) + refs = r.json() + self.assertEqual(refs, []) + + # draft with a normative reference + draft_bar = IndividualDraftFactory() + draft_bar = reload_db_objects(draft_bar) + RelatedDocument.objects.create( + source=draft, target=draft_bar, relationship_id="refnorm" + ) + url = urlreverse(viewname, kwargs={"doc_id": draft.id}) + r = self.client.get(url) + self.assertEqual(r.status_code, 403) + r = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(r.status_code, 200) + refs = r.json() + self.assertEqual(len(refs), 1) + self.assertEqual(refs[0]["id"], draft_bar.id) + self.assertEqual(refs[0]["name"], draft_bar.name) + + @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + @mock.patch("ietf.doc.tasks.signal_update_rfc_metadata_task.delay") + def test_notify_rfc_published(self, mock_task_delay): + url = urlreverse("ietf.api.purple_api.notify_rfc_published") + area = GroupFactory(type_id="area") + rfc_group = GroupFactory(type_id="wg") + draft_ad = RoleFactory(group=area, name_id="ad").person + rfc_ad = PersonFactory() + draft_authors = PersonFactory.create_batch(2) + rfc_authors = PersonFactory.create_batch(3) + draft = WgDraftFactory( + group__parent=area, authors=draft_authors, ad=draft_ad, stream_id="ietf" + ) + rfc_stream_id = "ise" + assert isinstance(draft, Document), "WgDraftFactory should generate a Document" + updates = RfcFactory.create_batch(2) + obsoletes = RfcFactory.create_batch(2) + unused_rfc_number = ( + Document.objects.filter(rfc_number__isnull=False).aggregate( + unused_rfc_number=Max("rfc_number") + 1 + )["unused_rfc_number"] + or 10000 + ) + + post_data = { + "published": "2025-12-17T20:29:00Z", + "draft_name": draft.name, + "draft_rev": draft.rev, + "rfc_number": unused_rfc_number, + "title": "RFC " + draft.title, + "authors": [ + { + "titlepage_name": f"titlepage {author.name}", + "is_editor": False, + "person": author.pk, + "email": author.email_address(), + "affiliation": "Some Affiliation", + "country": "CA", + } + for author in rfc_authors + ], + "group": rfc_group.acronym, + "stream": rfc_stream_id, + "abstract": "RFC version of " + draft.abstract, + "pages": draft.pages + 10, + "std_level": "ps", + "ad": rfc_ad.pk, + "obsoletes": [o.rfc_number for o in obsoletes], + "updates": [o.rfc_number for o in updates], + "subseries": [], + } + r = self.client.post(url, data=post_data, format="json") + self.assertEqual(r.status_code, 403) + + r = self.client.post( + url, data=post_data, format="json", headers={"X-Api-Key": "valid-token"} + ) + self.assertEqual(r.status_code, 200) + rfc = Document.objects.filter(rfc_number=unused_rfc_number).first() + self.assertIsNotNone(rfc) + self.assertEqual(rfc.came_from_draft(), draft) + self.assertEqual( + rfc.docevent_set.filter( + type="published_rfc", time="2025-12-17T20:29:00Z" + ).count(), + 1, + ) + self.assertEqual(rfc.title, "RFC " + draft.title) + self.assertEqual(rfc.documentauthor_set.count(), 0) + self.assertEqual( + [ + { + "titlepage_name": ra.titlepage_name, + "is_editor": ra.is_editor, + "person": ra.person, + "email": ra.email, + "affiliation": ra.affiliation, + "country": ra.country, + } + for ra in rfc.rfcauthor_set.all() + ], + [ + { + "titlepage_name": f"titlepage {author.name}", + "is_editor": False, + "person": author, + "email": author.email(), + "affiliation": "Some Affiliation", + "country": "CA", + } + for author in rfc_authors + ], + ) + self.assertEqual(rfc.group, rfc_group) + self.assertEqual(rfc.stream_id, rfc_stream_id) + self.assertEqual(rfc.abstract, "RFC version of " + draft.abstract) + self.assertEqual(rfc.pages, draft.pages + 10) + self.assertEqual(rfc.std_level_id, "ps") + self.assertEqual(rfc.ad, rfc_ad) + self.assertEqual(set(rfc.related_that_doc("obs")), set([o for o in obsoletes])) + self.assertEqual( + set(rfc.related_that_doc("updates")), set([o for o in updates]) + ) + self.assertEqual(rfc.part_of(), []) + self.assertEqual(draft.get_state().slug, "rfc") + # todo test non-empty relationships + # todo test references (when updating that is part of the handling) + + self.assertTrue(mock_task_delay.called) + mock_args, mock_kwargs = mock_task_delay.call_args + self.assertIn("rfc_number_list", mock_kwargs) + expected_rfc_number_list = [rfc.rfc_number] + expected_rfc_number_list.extend( + [d.rfc_number for d in updates + obsoletes] + ) + expected_rfc_number_list = sorted(set(expected_rfc_number_list)) + self.assertEqual(mock_kwargs["rfc_number_list"], expected_rfc_number_list) + + @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + def test_upload_rfc_files(self): + def _valid_post_data(): + """Generate a valid post data dict + + Each API call needs a fresh set of files, so don't reuse the return + value from this for multiple calls! + """ + return { + "rfc": rfc.rfc_number, + "contents": [ + ContentFile(b"This is .xml", "myfile.xml"), + ContentFile(b"This is .txt", "myfile.txt"), + ContentFile(b"This is .html", "myfile.html"), + ContentFile(b"This is .pdf", "myfile.pdf"), + ContentFile(b"This is .json", "myfile.json"), + ContentFile(b"This is .notprepped.xml", "myfile.notprepped.xml"), + ], + "replace": False, + } + + url = urlreverse("ietf.api.purple_api.upload_rfc_files") + unused_rfc_number = ( + Document.objects.filter(rfc_number__isnull=False).aggregate( + unused_rfc_number=Max("rfc_number") + 1 + )["unused_rfc_number"] + or 10000 + ) + + rfc = WgRfcFactory(rfc_number=unused_rfc_number) + assert isinstance(rfc, Document), "WgRfcFactory should generate a Document" + with TemporaryDirectory() as rfc_dir: + settings.RFC_PATH = rfc_dir # affects overridden settings + rfc_path = Path(rfc_dir) + (rfc_path / "prerelease").mkdir() + content = StringIO("XML content\n") + content.name = "myrfc.xml" + + # no api key + r = self.client.post(url, _valid_post_data(), format="multipart") + self.assertEqual(r.status_code, 403) + + # invalid RFC + r = self.client.post( + url, + _valid_post_data() | {"rfc": unused_rfc_number + 1}, + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 400) + + # empty files + r = self.client.post( + url, + _valid_post_data() | { + "contents": [ + ContentFile(b"", "myfile.xml"), + ContentFile(b"", "myfile.txt"), + ContentFile(b"", "myfile.html"), + ContentFile(b"", "myfile.pdf"), + ContentFile(b"", "myfile.json"), + ContentFile(b"", "myfile.notprepped.xml"), + ] + }, + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 400) + + # bad file type + r = self.client.post( + url, + _valid_post_data() | { + "contents": [ + ContentFile(b"Some content", "myfile.jpg"), + ] + }, + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 400) + + # Put a file in the way. Post should fail because replace = False + file_in_the_way = (rfc_path / f"rfc{unused_rfc_number}.txt") + file_in_the_way.touch() + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 409) # conflict + file_in_the_way.unlink() + + # Put a blob in the way. Post should fail because replace = False + blob_in_the_way = Blob.objects.create( + bucket="rfc", name=f"txt/rfc{unused_rfc_number}.txt", content=b"" + ) + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 409) # conflict + blob_in_the_way.delete() + + # valid post + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 200) + for extension in ["xml", "txt", "html", "pdf", "json"]: + filename = f"rfc{unused_rfc_number}.{extension}" + self.assertEqual( + (rfc_path / filename) + .read_text(), + f"This is .{extension}", + f"{extension} file should contain the expected content", + ) + self.assertEqual( + bytes( + Blob.objects.get( + bucket="rfc", name=f"{extension}/{filename}" + ).content + ), + f"This is .{extension}".encode("utf-8"), + f"{extension} blob should contain the expected content", + ) + # special case for notprepped + notprepped_fn = f"rfc{unused_rfc_number}.notprepped.xml" + self.assertEqual( + ( + rfc_path / "prerelease" / notprepped_fn + ).read_text(), + "This is .notprepped.xml", + ".notprepped.xml file should contain the expected content", + ) + self.assertEqual( + bytes( + Blob.objects.get( + bucket="rfc", name=f"notprepped/{notprepped_fn}" + ).content + ), + b"This is .notprepped.xml", + ".notprepped.xml blob should contain the expected content", + ) + + # re-post with replace = False should now fail + r = self.client.post( + url, + _valid_post_data(), + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 409) # conflict + + # re-post with replace = True should succeed + r = self.client.post( + url, + _valid_post_data() | {"replace": True}, + format="multipart", + headers={"X-Api-Key": "valid-token"}, + ) + self.assertEqual(r.status_code, 200) # conflict + + @override_settings(APP_API_TOKENS={"ietf.api.views_rpc": ["valid-token"]}) + @mock.patch("ietf.api.views_rpc.create_rfc_index_task") + def test_refresh_rfc_index(self, mock_task): + url = urlreverse("ietf.api.purple_api.refresh_rfc_index") + response = self.client.get(url) + self.assertEqual(response.status_code, 403) + response = self.client.get(url, headers={"X-Api-Key": "invalid-token"}) + self.assertEqual(response.status_code, 403) + response = self.client.get(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(response.status_code, 405) + self.assertFalse(mock_task.delay.called) + response = self.client.post(url, headers={"X-Api-Key": "valid-token"}) + self.assertEqual(response.status_code, 202) + self.assertTrue(mock_task.delay.called) diff --git a/ietf/api/urls.py b/ietf/api/urls.py index 6f2efb3c1e..7a082567b8 100644 --- a/ietf/api/urls.py +++ b/ietf/api/urls.py @@ -1,26 +1,31 @@ # Copyright The IETF Trust 2017-2024, All Rights Reserved +from drf_spectacular.views import SpectacularAPIView + from django.conf import settings -from django.urls import include +from django.urls import include, path from django.views.generic import TemplateView from ietf import api -from ietf.doc import views_ballot +from ietf.doc import views_ballot, api as doc_api from ietf.meeting import views as meeting_views from ietf.submit import views as submit_views from ietf.utils.urls import url from . import views as api_views +from .routers import PrefixedSimpleRouter # DRF API routing - disabled until we plan to use it -# from drf_spectacular.views import SpectacularAPIView -# from django.urls import path # from ietf.person import api as person_api -# from .routers import PrefixedSimpleRouter # core_router = PrefixedSimpleRouter(name_prefix="ietf.api.core_api") # core api router # core_router.register("email", person_api.EmailViewSet) # core_router.register("person", person_api.PersonViewSet) +# todo more general name for this API? +red_router = PrefixedSimpleRouter(name_prefix="ietf.api.red_api") # red api router +red_router.register("doc", doc_api.RfcViewSet) +red_router.register("subseries", doc_api.SubseriesViewSet, basename="subseries") + api.autodiscover() urlpatterns = [ @@ -32,7 +37,9 @@ url(r'^v2/person/person', api_views.ApiV2PersonExportView.as_view()), # --- DRF API --- # path("core/", include(core_router.urls)), - # path("schema/", SpectacularAPIView.as_view()), + path("purple/", include("ietf.api.urls_rpc")), + path("red/", include(red_router.urls)), + path("schema/", SpectacularAPIView.as_view()), # # --- Custom API endpoints, sorted alphabetically --- # Email alias information for drafts @@ -49,6 +56,9 @@ url(r'^group/role-holder-addresses/$', api_views.role_holder_addresses), # Let IESG members set positions programmatically url(r'^iesg/position', views_ballot.api_set_position), + # Find the blob to store for a given materials document path + url(r'^meeting/(?:(?P(?:interim-)?[a-z0-9-]+)/)?materials/%(document)s(?P\.[A-Za-z0-9]+)?/resolve-cached/$' % settings.URL_REGEXPS, meeting_views.api_resolve_materials_name_cached), + url(r'^meeting/blob/(?P[a-z0-9-]+)/(?P[a-z][a-z0-9.-]+)$', meeting_views.api_retrieve_materials_blob), # Let Meetecho set session video URLs url(r'^meeting/session/video/url$', meeting_views.api_set_session_video_url), # Let Meetecho tell us the name of its recordings diff --git a/ietf/api/urls_rpc.py b/ietf/api/urls_rpc.py new file mode 100644 index 0000000000..8555610dc3 --- /dev/null +++ b/ietf/api/urls_rpc.py @@ -0,0 +1,47 @@ +# Copyright The IETF Trust 2023-2026, All Rights Reserved +from django.urls import include, path + +from ietf.api import views_rpc +from ietf.api.routers import PrefixedDefaultRouter +from ietf.utils.urls import url + +router = PrefixedDefaultRouter(use_regex_path=False, name_prefix="ietf.api.purple_api") +router.include_format_suffixes = False +router.register(r"draft", views_rpc.DraftViewSet, basename="draft") +router.register(r"person", views_rpc.PersonViewSet) +router.register(r"rfc", views_rpc.RfcViewSet, basename="rfc") + +router.register( + r"rfc//authors", + views_rpc.RfcAuthorViewSet, + basename="rfc-authors", +) + +urlpatterns = [ + url(r"^doc/drafts_by_names/", views_rpc.DraftsByNamesView.as_view()), + url(r"^persons/search/", views_rpc.RpcPersonSearch.as_view()), + path( + r"rfc/publish/", + views_rpc.RfcPubNotificationView.as_view(), + name="ietf.api.purple_api.notify_rfc_published", + ), + path( + r"rfc/publish/files/", + views_rpc.RfcPubFilesView.as_view(), + name="ietf.api.purple_api.upload_rfc_files", + ), + path( + r"rfc_index/refresh/", + views_rpc.RfcIndexView.as_view(), + name="ietf.api.purple_api.refresh_rfc_index", + ), + path(r"subject//person/", views_rpc.SubjectPersonView.as_view()), +] + +# add routers at the end so individual routes can steal parts of their address +# space (e.g., ^rfc/publish/ superseding the ^rfc/ routes of RfcViewSet) +urlpatterns.extend( + [ + path("", include(router.urls)), + ] +) diff --git a/ietf/api/views.py b/ietf/api/views.py index 22523b2f17..420bc39693 100644 --- a/ietf/api/views.py +++ b/ietf/api/views.py @@ -97,7 +97,7 @@ class PersonalInformationExportView(DetailView, JsonExportMixin): def get(self, request): person = get_object_or_404(self.model, user=request.user) - expand = ['searchrule', 'documentauthor', 'ad_document_set', 'ad_dochistory_set', 'docevent', + expand = ['searchrule', 'documentauthor', 'rfcauthor', 'ad_document_set', 'ad_dochistory_set', 'docevent', 'ballotpositiondocevent', 'deletedevent', 'email_set', 'groupevent', 'role', 'rolehistory', 'iprdisclosurebase', 'iprevent', 'liaisonstatementevent', 'allowlisted', 'schedule', 'constraint', 'schedulingevent', 'message', 'sendqueue', 'nominee', 'topicfeedbacklastseen', 'alias', 'email', 'apikeys', 'personevent', diff --git a/ietf/api/views_rpc.py b/ietf/api/views_rpc.py new file mode 100644 index 0000000000..c7ae699005 --- /dev/null +++ b/ietf/api/views_rpc.py @@ -0,0 +1,534 @@ +# Copyright The IETF Trust 2023-2026, All Rights Reserved +import os +import shutil +from pathlib import Path +from tempfile import TemporaryDirectory + +from django.conf import settings +from django.db import IntegrityError +from drf_spectacular.utils import OpenApiParameter +from rest_framework import mixins, parsers, serializers, viewsets, status +from rest_framework.decorators import action +from rest_framework.exceptions import APIException +from rest_framework.views import APIView +from rest_framework.response import Response + +from django.db.models import CharField as ModelCharField, OuterRef, Subquery, Q +from django.db.models.functions import Coalesce +from django.http import Http404 +from drf_spectacular.utils import extend_schema_view, extend_schema +from rest_framework import generics +from rest_framework.fields import CharField as DrfCharField +from rest_framework.filters import SearchFilter +from rest_framework.pagination import LimitOffsetPagination + +from ietf.api.serializers_rpc import ( + PersonSerializer, + FullDraftSerializer, + DraftSerializer, + SubmittedToQueueSerializer, + OriginalStreamSerializer, + ReferenceSerializer, + EmailPersonSerializer, + RfcWithAuthorsSerializer, + DraftWithAuthorsSerializer, + NotificationAckSerializer, RfcPubSerializer, RfcFileSerializer, + EditableRfcSerializer, +) +from ietf.doc.models import Document, DocHistory, RfcAuthor, DocEvent +from ietf.doc.serializers import RfcAuthorSerializer +from ietf.doc.storage_utils import remove_from_storage, store_file, exists_in_storage +from ietf.doc.tasks import signal_update_rfc_metadata_task +from ietf.person.models import Email, Person +from ietf.sync.tasks import create_rfc_index_task + + +class Conflict(APIException): + status_code = status.HTTP_409_CONFLICT + default_detail = "Conflict." + default_code = "conflict" + + +@extend_schema_view( + retrieve=extend_schema( + operation_id="get_person_by_id", + summary="Find person by ID", + description="Returns a single person", + parameters=[ + OpenApiParameter( + name="person_id", + type=int, + location="path", + description="Person ID identifying this person.", + ), + ], + ), +) +class PersonViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet): + queryset = Person.objects.all() + serializer_class = PersonSerializer + api_key_endpoint = "ietf.api.views_rpc" + lookup_url_kwarg = "person_id" + + @extend_schema( + operation_id="get_persons", + summary="Get a batch of persons", + description="Returns a list of persons matching requested ids. Omits any that are missing.", + request=list[int], + responses=PersonSerializer(many=True), + ) + @action(detail=False, methods=["post"]) + def batch(self, request): + """Get a batch of rpc person names""" + pks = request.data + return Response( + self.get_serializer(Person.objects.filter(pk__in=pks), many=True).data + ) + + @extend_schema( + operation_id="persons_by_email", + summary="Get a batch of persons by email addresses", + description=( + "Returns a list of persons matching requested ids. " + "Omits any that are missing." + ), + request=list[str], + responses=EmailPersonSerializer(many=True), + ) + @action(detail=False, methods=["post"], serializer_class=EmailPersonSerializer) + def batch_by_email(self, request): + emails = Email.objects.filter(address__in=request.data, person__isnull=False) + serializer = self.get_serializer(emails, many=True) + return Response(serializer.data) + + +class SubjectPersonView(APIView): + api_key_endpoint = "ietf.api.views_rpc" + + @extend_schema( + operation_id="get_subject_person_by_id", + summary="Find person for OIDC subject by ID", + description="Returns a single person", + responses=PersonSerializer, + parameters=[ + OpenApiParameter( + name="subject_id", + type=str, + description="subject ID of person to return", + location="path", + ), + ], + ) + def get(self, request, subject_id: str): + try: + user_id = int(subject_id) + except ValueError: + raise serializers.ValidationError( + {"subject_id": "This field must be an integer value."} + ) + person = Person.objects.filter(user__pk=user_id).first() + if person: + return Response(PersonSerializer(person).data) + raise Http404 + + +class RpcLimitOffsetPagination(LimitOffsetPagination): + default_limit = 10 + max_limit = 100 + + +class SingleTermSearchFilter(SearchFilter): + """SearchFilter backend that does not split terms + + The default SearchFilter treats comma or whitespace-separated terms as individual + search terms. This backend instead searches for the exact term. + """ + + def get_search_terms(self, request): + value = request.query_params.get(self.search_param, "") + field = DrfCharField(trim_whitespace=False, allow_blank=True) + cleaned_value = field.run_validation(value) + return [cleaned_value] + + +@extend_schema_view( + get=extend_schema( + operation_id="search_person", + description="Get a list of persons, matching by partial name or email", + ), +) +class RpcPersonSearch(generics.ListAPIView): + # n.b. the OpenAPI schema for this can be generated by running + # ietf/manage.py spectacular --file spectacular.yaml + # and extracting / touching up the rpc_person_search_list operation + api_key_endpoint = "ietf.api.views_rpc" + queryset = Person.objects.all() + serializer_class = PersonSerializer + pagination_class = RpcLimitOffsetPagination + + # Searchable on all name-like fields or email addresses + filter_backends = [SingleTermSearchFilter] + search_fields = ["name", "plain", "email__address"] + + +@extend_schema_view( + retrieve=extend_schema( + operation_id="get_draft_by_id", + summary="Get a draft", + description="Returns the draft for the requested ID", + parameters=[ + OpenApiParameter( + name="doc_id", + type=int, + location="path", + description="Doc ID identifying this draft.", + ), + ], + ), + submitted_to_rpc=extend_schema( + operation_id="submitted_to_rpc", + summary="List documents ready to enter the RFC Editor Queue", + description="List documents ready to enter the RFC Editor Queue", + responses=SubmittedToQueueSerializer(many=True), + ), +) +class DraftViewSet(mixins.RetrieveModelMixin, viewsets.GenericViewSet): + queryset = Document.objects.filter(type_id="draft") + serializer_class = FullDraftSerializer + api_key_endpoint = "ietf.api.views_rpc" + lookup_url_kwarg = "doc_id" + + @action(detail=False, serializer_class=SubmittedToQueueSerializer) + def submitted_to_rpc(self, request): + """Return documents in datatracker that have been submitted to the RPC but are not yet in the queue + + Those queries overreturn - there may be things, particularly not from the IETF stream that are already in the queue. + """ + ietf_docs = Q(states__type_id="draft-iesg", states__slug__in=["ann"]) + irtf_iab_ise_docs = Q( + states__type_id__in=[ + "draft-stream-iab", + "draft-stream-irtf", + "draft-stream-ise", + ], + states__slug__in=["rfc-edit"], + ) + # TODO: Need a way to talk about editorial stream docs + docs = ( + self.get_queryset() + .filter(type_id="draft") + .filter(ietf_docs | irtf_iab_ise_docs) + ) + serializer = self.get_serializer(docs, many=True) + return Response(serializer.data) + + @extend_schema( + operation_id="get_draft_references", + summary="Get normative references to I-Ds", + description=( + "Returns the id and name of each normatively " + "referenced Internet-Draft for the given docId" + ), + parameters=[ + OpenApiParameter( + name="doc_id", + type=int, + location="path", + description="Doc ID identifying this draft.", + ), + ], + responses=ReferenceSerializer(many=True), + ) + @action(detail=True, serializer_class=ReferenceSerializer) + def references(self, request, doc_id=None): + doc = self.get_object() + serializer = self.get_serializer( + [ + reference + for reference in doc.related_that_doc("refnorm") + if reference.type_id == "draft" + ], + many=True, + ) + return Response(serializer.data) + + @extend_schema( + operation_id="get_draft_authors", + summary="Gather authors of the drafts with the given names", + description="returns a list mapping draft names to objects describing authors", + request=list[str], + responses=DraftWithAuthorsSerializer(many=True), + ) + @action(detail=False, methods=["post"], serializer_class=DraftWithAuthorsSerializer) + def bulk_authors(self, request): + drafts = self.get_queryset().filter(name__in=request.data) + serializer = self.get_serializer(drafts, many=True) + return Response(serializer.data) + + +@extend_schema_view( + rfc_original_stream=extend_schema( + operation_id="get_rfc_original_streams", + summary="Get the streams RFCs were originally published into", + description="returns a list of dicts associating an RFC with its originally published stream", + responses=OriginalStreamSerializer(many=True), + ) +) +class RfcViewSet(mixins.UpdateModelMixin, viewsets.GenericViewSet): + queryset = Document.objects.filter(type_id="rfc") + api_key_endpoint = "ietf.api.views_rpc" + lookup_field = "rfc_number" + serializer_class = EditableRfcSerializer + + def perform_update(self, serializer): + DocEvent.objects.create( + doc=serializer.instance, + rev=serializer.instance.rev, + by=Person.objects.get(name="(System)"), + type="sync_from_rfc_editor", + desc="Metadata update from RFC Editor", + ) + super().perform_update(serializer) + + @action(detail=False, serializer_class=OriginalStreamSerializer) + def rfc_original_stream(self, request): + rfcs = self.get_queryset().annotate( + orig_stream_id=Coalesce( + Subquery( + DocHistory.objects.filter(doc=OuterRef("pk")) + .exclude(stream__isnull=True) + .order_by("time") + .values_list("stream_id", flat=True)[:1] + ), + "stream_id", + output_field=ModelCharField(), + ), + ) + serializer = self.get_serializer(rfcs, many=True) + return Response(serializer.data) + + @extend_schema( + operation_id="get_rfc_authors", + summary="Gather authors of the RFCs with the given numbers", + description="returns a list mapping rfc numbers to objects describing authors", + request=list[int], + responses=RfcWithAuthorsSerializer(many=True), + ) + @action(detail=False, methods=["post"], serializer_class=RfcWithAuthorsSerializer) + def bulk_authors(self, request): + rfcs = self.get_queryset().filter(rfc_number__in=request.data) + serializer = self.get_serializer(rfcs, many=True) + return Response(serializer.data) + + +class DraftsByNamesView(APIView): + api_key_endpoint = "ietf.api.views_rpc" + + @extend_schema( + operation_id="get_drafts_by_names", + summary="Get a batch of drafts by draft names", + description="returns a list of drafts with matching names", + request=list[str], + responses=DraftSerializer(many=True), + ) + def post(self, request): + names = request.data + docs = Document.objects.filter(type_id="draft", name__in=names) + return Response(DraftSerializer(docs, many=True).data) + + +class RfcAuthorViewSet(viewsets.ReadOnlyModelViewSet): + """ViewSet for RfcAuthor model + + Router needs to provide rfc_number as a kwarg + """ + api_key_endpoint = "ietf.api.views_rpc" + + queryset = RfcAuthor.objects.all() + serializer_class = RfcAuthorSerializer + lookup_url_kwarg = "author_id" + rfc_number_param = "rfc_number" + + def get_queryset(self): + return ( + super() + .get_queryset() + .filter( + document__type_id="rfc", + document__rfc_number=self.kwargs[self.rfc_number_param], + ) + ) + + +class RfcPubNotificationView(APIView): + api_key_endpoint = "ietf.api.views_rpc" + + @extend_schema( + operation_id="notify_rfc_published", + summary="Notify datatracker of RFC publication", + request=RfcPubSerializer, + responses=NotificationAckSerializer, + ) + def post(self, request): + serializer = RfcPubSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + # Create RFC + try: + rfc = serializer.save() + except IntegrityError as err: + if Document.objects.filter( + rfc_number=serializer.validated_data["rfc_number"] + ): + raise serializers.ValidationError( + "RFC with that number already exists", + code="rfc-number-in-use", + ) + raise serializers.ValidationError( + f"Unable to publish: {err}", + code="unknown-integrity-error", + ) + rfc_number_list = [rfc.rfc_number] + rfc_number_list.extend( + [d.rfc_number for d in rfc.related_that_doc(("updates", "obs"))] + ) + rfc_number_list = sorted(set(rfc_number_list)) + signal_update_rfc_metadata_task.delay(rfc_number_list=rfc_number_list) + return Response(NotificationAckSerializer().data) + + +class RfcPubFilesView(APIView): + api_key_endpoint = "ietf.api.views_rpc" + parser_classes = [parsers.MultiPartParser] + + def _fs_destination(self, filename: str | Path) -> Path: + """Destination for an uploaded RFC file in the filesystem + + Strips any path components in filename and returns an absolute Path. + """ + rfc_path = Path(settings.RFC_PATH) + filename = Path(filename) # could potentially have directory components + extension = "".join(filename.suffixes) + if extension == ".notprepped.xml": + return rfc_path / "prerelease" / filename.name + return rfc_path / filename.name + + def _blob_destination(self, filename: str | Path) -> str: + """Destination name for an uploaded RFC file in the blob store + + Strips any path components in filename and returns an absolute Path. + """ + filename = Path(filename) # could potentially have directory components + extension = "".join(filename.suffixes) + if extension == ".notprepped.xml": + file_type = "notprepped" + elif extension[0] == ".": + file_type = extension[1:] + else: + raise serializers.ValidationError( + f"Extension does not begin with '.'!? ({filename})", + ) + return f"{file_type}/{filename.name}" + + @extend_schema( + operation_id="upload_rfc_files", + summary="Upload files for a published RFC", + request=RfcFileSerializer, + responses=NotificationAckSerializer, + ) + def post(self, request): + serializer = RfcFileSerializer( + # many=True, + data=request.data, + ) + serializer.is_valid(raise_exception=True) + rfc = serializer.validated_data["rfc"] + uploaded_files = serializer.validated_data["contents"] # list[UploadedFile] + replace = serializer.validated_data["replace"] + dest_stem = f"rfc{rfc.rfc_number}" + mtime = serializer.validated_data["mtime"] + mtimestamp = mtime.timestamp() + blob_kind = "rfc" + + # List of files that might exist for an RFC + possible_rfc_files = [ + self._fs_destination(dest_stem + ext) + for ext in serializer.allowed_extensions + ] + possible_rfc_blobs = [ + self._blob_destination(dest_stem + ext) + for ext in serializer.allowed_extensions + ] + if not replace: + # this is the default: refuse to overwrite anything if not replacing + for possible_existing_file in possible_rfc_files: + if possible_existing_file.exists(): + raise Conflict( + "File(s) already exist for this RFC", + code="files-exist", + ) + for possible_existing_blob in possible_rfc_blobs: + if exists_in_storage( + kind=blob_kind, name=possible_existing_blob + ): + raise Conflict( + "Blob(s) already exist for this RFC", + code="blobs-exist", + ) + + with TemporaryDirectory() as tempdir: + # Save files in a temporary directory. Use the uploaded filename + # extensions to identify files, but ignore the stems and generate our own. + files_to_move = [] # list[Path] + tmpfile_stem = Path(tempdir) / dest_stem + for upfile in uploaded_files: + uploaded_filename = Path(upfile.name) # name supplied by request + uploaded_ext = "".join(uploaded_filename.suffixes) + tempfile_path = tmpfile_stem.with_suffix(uploaded_ext) + with tempfile_path.open("wb") as dest: + for chunk in upfile.chunks(): + dest.write(chunk) + os.utime(tempfile_path, (mtimestamp, mtimestamp)) + files_to_move.append(tempfile_path) + # copy files to final location, removing any existing ones first if the + # remove flag was set + if replace: + for possible_existing_file in possible_rfc_files: + possible_existing_file.unlink(missing_ok=True) + for possible_existing_blob in possible_rfc_blobs: + remove_from_storage( + blob_kind, possible_existing_blob, warn_if_missing=False + ) + for ftm in files_to_move: + with ftm.open("rb") as f: + store_file( + kind=blob_kind, + name=self._blob_destination(ftm), + file=f, + doc_name=rfc.name, + doc_rev=rfc.rev, # expect blank, but match whatever it is + mtime=mtime, + ) + destination = self._fs_destination(ftm) + if ( + settings.SERVER_MODE != "production" + and not destination.parent.exists() + ): + destination.parent.mkdir() + shutil.move(ftm, destination) + + return Response(NotificationAckSerializer().data) + + +class RfcIndexView(APIView): + api_key_endpoint = "ietf.api.views_rpc" + + @extend_schema( + operation_id="refresh_rfc_index", + summary="Refresh rfc-index files", + description="Requests creation of rfc-index.xml and rfc-index.txt files", + responses={202: None}, + request=None, + ) + def post(self, request): + create_rfc_index_task.delay() + return Response(status=202) diff --git a/ietf/bin/aliases-from-json.py b/ietf/bin/aliases-from-json.py index a0c383a1ac..0da5d1f8b9 100644 --- a/ietf/bin/aliases-from-json.py +++ b/ietf/bin/aliases-from-json.py @@ -38,7 +38,7 @@ def generate_files(records, adest, vdest, postconfirm, vdomain): vpath = tmppath / "virtual" with apath.open("w") as afile, vpath.open("w") as vfile: - date = datetime.datetime.now(datetime.timezone.utc) + date = datetime.datetime.now(datetime.UTC) signature = f"# Generated by {Path(__file__).absolute()} at {date}\n" afile.write(signature) vfile.write(signature) diff --git a/ietf/blobdb/admin.py b/ietf/blobdb/admin.py index f4cd002e07..3e1a2a311f 100644 --- a/ietf/blobdb/admin.py +++ b/ietf/blobdb/admin.py @@ -3,7 +3,7 @@ from django.db.models.functions import Length from rangefilter.filters import DateRangeQuickSelectListFilterBuilder -from .models import Blob +from .models import Blob, ResolvedMaterial @admin.register(Blob) @@ -29,3 +29,12 @@ def get_queryset(self, request): def object_size(self, instance): """Get the size of the object""" return instance.object_size # annotation added in get_queryset() + + +@admin.register(ResolvedMaterial) +class ResolvedMaterialAdmin(admin.ModelAdmin): + model = ResolvedMaterial + list_display = ["name", "meeting_number", "bucket", "blob"] + list_filter = ["meeting_number", "bucket"] + search_fields = ["name", "blob"] + ordering = ["name"] diff --git a/ietf/blobdb/migrations/0002_resolvedmaterial.py b/ietf/blobdb/migrations/0002_resolvedmaterial.py new file mode 100644 index 0000000000..e0ab405b11 --- /dev/null +++ b/ietf/blobdb/migrations/0002_resolvedmaterial.py @@ -0,0 +1,48 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("blobdb", "0001_initial"), + ] + + operations = [ + migrations.CreateModel( + name="ResolvedMaterial", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(help_text="Name to resolve", max_length=300)), + ( + "meeting_number", + models.CharField( + help_text="Meeting material is related to", max_length=64 + ), + ), + ( + "bucket", + models.CharField(help_text="Resolved bucket name", max_length=255), + ), + ( + "blob", + models.CharField(help_text="Resolved blob name", max_length=300), + ), + ], + ), + migrations.AddConstraint( + model_name="resolvedmaterial", + constraint=models.UniqueConstraint( + fields=("name", "meeting_number"), name="unique_name_per_meeting" + ), + ), + ] diff --git a/ietf/blobdb/models.py b/ietf/blobdb/models.py index 8f423d9f6c..27325ada5d 100644 --- a/ietf/blobdb/models.py +++ b/ietf/blobdb/models.py @@ -64,6 +64,9 @@ class Meta: ), ] + def __str__(self): + return f"{self.bucket}:{self.name}" + def save(self, **kwargs): db = get_blobdb() with transaction.atomic(using=db): @@ -96,3 +99,23 @@ def _emit_blob_change_event(self, using=None): ), using=using, ) + + +class ResolvedMaterial(models.Model): + # A Document name can be 255 characters; allow this name to be a bit longer + name = models.CharField(max_length=300, help_text="Name to resolve") + meeting_number = models.CharField( + max_length=64, help_text="Meeting material is related to" + ) + bucket = models.CharField(max_length=255, help_text="Resolved bucket name") + blob = models.CharField(max_length=300, help_text="Resolved blob name") + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["name", "meeting_number"], name="unique_name_per_meeting" + ) + ] + + def __str__(self): + return f"{self.name}@{self.meeting_number} -> {self.bucket}:{self.blob}" diff --git a/ietf/blobdb/replication.py b/ietf/blobdb/replication.py index b9d55c9498..d251d3b95c 100644 --- a/ietf/blobdb/replication.py +++ b/ietf/blobdb/replication.py @@ -146,11 +146,11 @@ def replicate_blob(bucket, name): blob = fetch_blob_via_sql(bucket, name) if blob is None: if verbose_logging_enabled(): - log.log("Deleting {bucket}:{name} from replica") + log.log(f"Deleting {bucket}:{name} from replica") try: destination_storage.delete(name) except Exception as e: - log.log("Failed to delete {bucket}:{name} from replica: {e}") + log.log(f"Failed to delete {bucket}:{name} from replica: {e}") raise ReplicationError from e else: # Add metadata expected by the MetadataS3Storage @@ -170,7 +170,7 @@ def replicate_blob(bucket, name): try: destination_storage.save(name, file_with_metadata) except Exception as e: - log.log("Failed to save {bucket}:{name} to replica: {e}") + log.log(f"Failed to save {bucket}:{name} to replica: {e}") raise ReplicationError from e diff --git a/ietf/community/utils.py b/ietf/community/utils.py index f23e8d26ab..b6137095ef 100644 --- a/ietf/community/utils.py +++ b/ietf/community/utils.py @@ -72,8 +72,10 @@ def docs_matching_community_list_rule(rule): return docs.filter(group=rule.group_id) elif rule.rule_type.startswith("state_"): return docs - elif rule.rule_type in ["author", "author_rfc"]: + elif rule.rule_type == "author": return docs.filter(documentauthor__person=rule.person) + elif rule.rule_type == "author_rfc": + return docs.filter(Q(rfcauthor__person=rule.person)|Q(rfcauthor__isnull=True,documentauthor__person=rule.person)) elif rule.rule_type == "ad": return docs.filter(ad=rule.person) elif rule.rule_type == "shepherd": @@ -122,9 +124,16 @@ def community_list_rules_matching_doc(doc): # author rules if doc.type_id == "rfc": + has_rfcauthors = doc.rfcauthor_set.exists() rules |= SearchRule.objects.filter( rule_type="author_rfc", - person__in=list(Person.objects.filter(documentauthor__document=doc)), + person__in=list( + Person.objects.filter( + Q(rfcauthor__document=doc) + if has_rfcauthors + else Q(documentauthor__document=doc) + ) + ), ) else: rules |= SearchRule.objects.filter( diff --git a/ietf/context_processors.py b/ietf/context_processors.py index baa8d7a5d2..5aaa4ab256 100644 --- a/ietf/context_processors.py +++ b/ietf/context_processors.py @@ -5,6 +5,7 @@ from django.conf import settings from django.utils import timezone from ietf import __version__, __patch__, __release_branch__, __release_hash__ +from opentelemetry.propagate import inject def server_mode(request): return {'server_mode': settings.SERVER_MODE} @@ -51,3 +52,8 @@ def timezone_now(request): return { 'timezone_now': timezone.now(), } + +def traceparent_id(request): + context_extras = {} + inject(context_extras) + return { "otel": context_extras } diff --git a/ietf/doc/admin.py b/ietf/doc/admin.py index b492aa3423..0d04e8db3a 100644 --- a/ietf/doc/admin.py +++ b/ietf/doc/admin.py @@ -13,8 +13,10 @@ TelechatDocEvent, BallotPositionDocEvent, ReviewRequestDocEvent, InitialReviewDocEvent, AddedMessageEvent, SubmissionDocEvent, DeletedEvent, EditedAuthorsDocEvent, DocumentURL, ReviewAssignmentDocEvent, IanaExpertDocEvent, IRSGBallotDocEvent, DocExtResource, DocumentActionHolder, - BofreqEditorDocEvent, BofreqResponsibleDocEvent, StoredObject ) + BofreqEditorDocEvent, BofreqResponsibleDocEvent, StoredObject, RfcAuthor, + EditedRfcAuthorsDocEvent) +from ietf.utils.admin import SaferTabularInline from ietf.utils.validators import validate_external_resource_value class StateTypeAdmin(admin.ModelAdmin): @@ -22,23 +24,23 @@ class StateTypeAdmin(admin.ModelAdmin): admin.site.register(StateType, StateTypeAdmin) class StateAdmin(admin.ModelAdmin): - list_display = ["slug", "type", 'name', 'order', 'desc'] - list_filter = ["type", ] + list_display = ["slug", "type", 'name', 'order', 'desc', "used"] + list_filter = ["type", "used"] search_fields = ["slug", "type__label", "type__slug", "name", "desc"] filter_horizontal = ["next_states"] admin.site.register(State, StateAdmin) -class DocAuthorInline(admin.TabularInline): +class DocAuthorInline(SaferTabularInline): model = DocumentAuthor raw_id_fields = ['person', 'email'] extra = 1 -class DocActionHolderInline(admin.TabularInline): +class DocActionHolderInline(SaferTabularInline): model = DocumentActionHolder raw_id_fields = ['person'] extra = 1 -class RelatedDocumentInline(admin.TabularInline): +class RelatedDocumentInline(SaferTabularInline): model = RelatedDocument fk_name= 'source' def this(self, instance): @@ -48,7 +50,7 @@ def this(self, instance): raw_id_fields = ['target'] extra = 1 -class AdditionalUrlInLine(admin.TabularInline): +class AdditionalUrlInLine(SaferTabularInline): model = DocumentURL fields = ['tag','desc','url',] extra = 1 @@ -173,6 +175,7 @@ def short_desc(self, obj): admin.site.register(TelechatDocEvent, DocEventAdmin) admin.site.register(InitialReviewDocEvent, DocEventAdmin) admin.site.register(EditedAuthorsDocEvent, DocEventAdmin) +admin.site.register(EditedRfcAuthorsDocEvent, DocEventAdmin) admin.site.register(IanaExpertDocEvent, DocEventAdmin) class BallotPositionDocEventAdmin(DocEventAdmin): @@ -236,3 +239,11 @@ def is_deleted(self, instance): admin.site.register(StoredObject, StoredObjectAdmin) + +class RfcAuthorAdmin(admin.ModelAdmin): + # the email field in the list_display/readonly_fields works through a @property + list_display = ['id', 'document', 'titlepage_name', 'person', 'email', 'affiliation', 'country', 'order'] + search_fields = ['document__name', 'titlepage_name', 'person__name', 'person__email__address', 'affiliation', 'country'] + raw_id_fields = ["document", "person"] + readonly_fields = ["email"] +admin.site.register(RfcAuthor, RfcAuthorAdmin) diff --git a/ietf/doc/api.py b/ietf/doc/api.py new file mode 100644 index 0000000000..73fff6b27f --- /dev/null +++ b/ietf/doc/api.py @@ -0,0 +1,213 @@ +# Copyright The IETF Trust 2024-2026, All Rights Reserved +"""Doc API implementations""" + +from django.db.models import ( + BooleanField, + Count, + OuterRef, + Prefetch, + Q, + QuerySet, + Subquery, +) +from django.db.models.functions import TruncDate +from django_filters import rest_framework as filters +from rest_framework import filters as drf_filters +from rest_framework.mixins import ListModelMixin, RetrieveModelMixin +from rest_framework.pagination import LimitOffsetPagination +from rest_framework.viewsets import GenericViewSet + +from ietf.group.models import Group +from ietf.name.models import StreamName, DocTypeName +from ietf.utils.timezone import RPC_TZINFO +from .models import ( + Document, + DocEvent, + RelatedDocument, + DocumentAuthor, + SUBSERIES_DOC_TYPE_IDS, +) +from .serializers import ( + RfcMetadataSerializer, + RfcStatus, + RfcSerializer, + SubseriesDocSerializer, +) + + +class RfcLimitOffsetPagination(LimitOffsetPagination): + default_limit = 10 + max_limit = 500 + + +class NumberInFilter(filters.BaseInFilter, filters.NumberFilter): + """Filter against a comma-separated list of numbers""" + pass + + +class RfcFilter(filters.FilterSet): + published = filters.DateFromToRangeFilter() + stream = filters.ModelMultipleChoiceFilter( + queryset=StreamName.objects.filter(used=True) + ) + number = NumberInFilter( + field_name="rfc_number" + ) + group = filters.ModelMultipleChoiceFilter( + queryset=Group.objects.all(), + field_name="group__acronym", + to_field_name="acronym", + ) + area = filters.ModelMultipleChoiceFilter( + queryset=Group.objects.areas(), + field_name="group__parent__acronym", + to_field_name="acronym", + ) + status = filters.MultipleChoiceFilter( + choices=[(slug, slug) for slug in RfcStatus.status_slugs], + method=RfcStatus.filter, + ) + sort = filters.OrderingFilter( + fields=( + ("rfc_number", "number"), # ?sort=number / ?sort=-number + ("published", "published"), # ?sort=published / ?sort=-published + ), + ) + + +class PrefetchRelatedDocument(Prefetch): + """Prefetch via a RelatedDocument + + Prefetches following RelatedDocument relationships to other docs. By default, includes + those for which the current RFC is the `source`. If `reverse` is True, includes those + for which it is the `target` instead. Defaults to only "rfc" documents. + """ + + @staticmethod + def _get_queryset(relationship_id, reverse, doc_type_ids): + """Get queryset to use for the prefetch""" + if isinstance(doc_type_ids, str): + doc_type_ids = (doc_type_ids,) + + return RelatedDocument.objects.filter( + **{ + "relationship_id": relationship_id, + f"{'source' if reverse else 'target'}__type_id__in": doc_type_ids, + } + ).select_related("source" if reverse else "target") + + def __init__(self, to_attr, relationship_id, reverse=False, doc_type_ids="rfc"): + super().__init__( + lookup="targets_related" if reverse else "relateddocument_set", + queryset=self._get_queryset(relationship_id, reverse, doc_type_ids), + to_attr=to_attr, + ) + + +def augment_rfc_queryset(queryset: QuerySet[Document]): + return ( + queryset.select_related("std_level", "stream") + .prefetch_related( + Prefetch( + "group", + Group.objects.select_related("parent"), + ), + Prefetch( + "documentauthor_set", + DocumentAuthor.objects.select_related("email", "person"), + ), + PrefetchRelatedDocument( + to_attr="drafts", + relationship_id="became_rfc", + doc_type_ids="draft", + reverse=True, + ), + PrefetchRelatedDocument(to_attr="obsoletes", relationship_id="obs"), + PrefetchRelatedDocument( + to_attr="obsoleted_by", relationship_id="obs", reverse=True + ), + PrefetchRelatedDocument(to_attr="updates", relationship_id="updates"), + PrefetchRelatedDocument( + to_attr="updated_by", relationship_id="updates", reverse=True + ), + PrefetchRelatedDocument( + to_attr="subseries", + relationship_id="contains", + reverse=True, + doc_type_ids=SUBSERIES_DOC_TYPE_IDS, + ), + ) + .annotate( + published_datetime=Subquery( + DocEvent.objects.filter( + doc_id=OuterRef("pk"), + type="published_rfc", + ) + .order_by("-time") + .values("time")[:1] + ), + ) + .annotate(published=TruncDate("published_datetime", tzinfo=RPC_TZINFO)) + .annotate( + # Count of "verified-errata" tags will be 1 or 0, convert to Boolean + has_errata=Count( + "tags", + filter=Q( + tags__slug="verified-errata", + ), + output_field=BooleanField(), + ) + ) + ) + + +class RfcViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): + api_key_endpoint = "ietf.api.red_api" # matches prefix in ietf/api/urls.py + lookup_field = "rfc_number" + queryset = augment_rfc_queryset( + Document.objects.filter(type_id="rfc", rfc_number__isnull=False) + ).order_by("-rfc_number") + + pagination_class = RfcLimitOffsetPagination + filter_backends = [filters.DjangoFilterBackend, drf_filters.SearchFilter] + filterset_class = RfcFilter + search_fields = ["title", "abstract"] + + def get_serializer_class(self): + if self.action == "retrieve": + return RfcSerializer + return RfcMetadataSerializer + + +class PrefetchSubseriesContents(Prefetch): + def __init__(self, to_attr): + super().__init__( + lookup="relateddocument_set", + queryset=RelatedDocument.objects.filter( + relationship_id="contains", + target__type_id="rfc", + ).prefetch_related( + Prefetch( + "target", + queryset=augment_rfc_queryset(Document.objects.all()), + ) + ), + to_attr=to_attr, + ) + + +class SubseriesFilter(filters.FilterSet): + type = filters.ModelMultipleChoiceFilter( + queryset=DocTypeName.objects.filter(pk__in=SUBSERIES_DOC_TYPE_IDS) + ) + + +class SubseriesViewSet(ListModelMixin, RetrieveModelMixin, GenericViewSet): + api_key_endpoint = "ietf.api.red_api" # matches prefix in ietf/api/urls.py + lookup_field = "name" + serializer_class = SubseriesDocSerializer + queryset = Document.objects.subseries_docs().prefetch_related( + PrefetchSubseriesContents(to_attr="contents") + ) + filter_backends = [filters.DjangoFilterBackend] + filterset_class = SubseriesFilter diff --git a/ietf/doc/expire.py b/ietf/doc/expire.py index bf8523aa98..d42af628f8 100644 --- a/ietf/doc/expire.py +++ b/ietf/doc/expire.py @@ -38,22 +38,46 @@ def expirable_drafts(queryset=None): # Populate this first time through (but after django has been set up) if nonexpirable_states is None: # all IESG states except I-D Exists and Dead block expiry - nonexpirable_states = list(State.objects.filter(used=True, type="draft-iesg").exclude(slug__in=("idexists", "dead"))) + nonexpirable_states = list( + State.objects.filter(used=True, type="draft-iesg").exclude( + slug__in=("idexists", "dead") + ) + ) # sent to RFC Editor and RFC Published block expiry (the latter # shouldn't be possible for an active draft, though) - nonexpirable_states += list(State.objects.filter(used=True, type__in=("draft-stream-iab", "draft-stream-irtf", "draft-stream-ise"), slug__in=("rfc-edit", "pub"))) + nonexpirable_states += list( + State.objects.filter( + used=True, + type__in=( + "draft-stream-iab", + "draft-stream-irtf", + "draft-stream-ise", + "draft-stream-editorial", + ), + slug__in=("rfc-edit", "pub"), + ) + ) # other IRTF states that block expiration - nonexpirable_states += list(State.objects.filter(used=True, type_id="draft-stream-irtf", slug__in=("irsgpoll", "iesg-rev",))) - - return queryset.filter( - states__type="draft", states__slug="active" - ).exclude( - expires=None - ).exclude( - states__in=nonexpirable_states - ).exclude( - tags="rfc-rev" # under review by the RFC Editor blocks expiry - ).distinct() + nonexpirable_states += list( + State.objects.filter( + used=True, + type_id="draft-stream-irtf", + slug__in=( + "irsgpoll", + "iesg-rev", + ), + ) + ) + + return ( + queryset.filter(states__type="draft", states__slug="active") + .exclude(expires=None) + .exclude(states__in=nonexpirable_states) + .exclude( + tags="rfc-rev" # under review by the RFC Editor blocks expiry + ) + .distinct() + ) def get_soon_to_expire_drafts(days_of_warning): diff --git a/ietf/doc/factories.py b/ietf/doc/factories.py index 19aa9ecc9c..1a178c6f31 100644 --- a/ietf/doc/factories.py +++ b/ietf/doc/factories.py @@ -14,7 +14,7 @@ from ietf.doc.models import ( Document, DocEvent, NewRevisionDocEvent, State, DocumentAuthor, StateDocEvent, BallotPositionDocEvent, BallotDocEvent, BallotType, IRSGBallotDocEvent, TelechatDocEvent, - DocumentActionHolder, BofreqEditorDocEvent, BofreqResponsibleDocEvent, DocExtResource ) + DocumentActionHolder, BofreqEditorDocEvent, BofreqResponsibleDocEvent, DocExtResource, RfcAuthor ) from ietf.group.models import Group from ietf.person.factories import PersonFactory from ietf.group.factories import RoleFactory @@ -311,6 +311,12 @@ class Meta: def desc(self): return 'New version available %s-%s'%(self.doc.name,self.rev) +class PublishedRfcDocEventFactory(DocEventFactory): + class Meta: + model = DocEvent + type = "published_rfc" + doc = factory.SubFactory(WgRfcFactory) + class StateDocEventFactory(DocEventFactory): class Meta: model = StateDocEvent @@ -382,6 +388,18 @@ class Meta: country = factory.Faker('country') order = factory.LazyAttribute(lambda o: o.document.documentauthor_set.count() + 1) +class RfcAuthorFactory(factory.django.DjangoModelFactory): + class Meta: + model = RfcAuthor + + document = factory.SubFactory(DocumentFactory) + titlepage_name = factory.LazyAttribute( + lambda obj: " ".join([obj.person.initials(), obj.person.last_name()]) + ) + person = factory.SubFactory('ietf.person.factories.PersonFactory') + affiliation = factory.Faker('company') + order = factory.LazyAttribute(lambda o: o.document.rfcauthor_set.count() + 1) + class WgDocumentAuthorFactory(DocumentAuthorFactory): document = factory.SubFactory(WgDraftFactory) diff --git a/ietf/doc/feeds.py b/ietf/doc/feeds.py index 500ed3cb18..afe96cf0df 100644 --- a/ietf/doc/feeds.py +++ b/ietf/doc/feeds.py @@ -1,5 +1,4 @@ -# Copyright The IETF Trust 2007-2020, All Rights Reserved -# -*- coding: utf-8 -*- +# Copyright The IETF Trust 2007-2026, All Rights Reserved import debug # pyflakes:ignore @@ -263,9 +262,11 @@ def item_extra_kwargs(self, item): ) extra.update({"media_contents": media_contents}) - extra.update({"doi": "10.17487/%s" % item.name.upper()}) extra.update( - {"doiuri": "http://dx.doi.org/10.17487/%s" % item.name.upper()} + { + "doi": item.doi, + "doiuri": f"https://doi.org/{item.doi}", + } ) # R104 Publisher (Mandatory - but we need a string from them first) diff --git a/ietf/doc/mails.py b/ietf/doc/mails.py index f20d398c3c..ddecbb6b54 100644 --- a/ietf/doc/mails.py +++ b/ietf/doc/mails.py @@ -103,61 +103,6 @@ def email_stream_changed(request, doc, old_stream, new_stream, text=""): dict(text=text, url=settings.IDTRACKER_BASE_URL + doc.get_absolute_url()), cc=cc) - -def email_wg_call_for_adoption_issued(request, doc, cfa_duration_weeks=None): - if cfa_duration_weeks is None: - cfa_duration_weeks=2 - (to, cc) = gather_address_lists("doc_wg_call_for_adoption_issued", doc=doc) - frm = request.user.person.formatted_email() - - end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=7 * cfa_duration_weeks) - - subject = f"Call for adoption: {doc.name}-{doc.rev} (Ends {end_date})" - - send_mail( - request, - to, - frm, - subject, - "doc/mail/wg_call_for_adoption_issued.txt", - dict( - doc=doc, - subject=subject, - url=settings.IDTRACKER_BASE_URL + doc.get_absolute_url(), - end_date=end_date, - cfa_duration_weeks=cfa_duration_weeks, - wg_list=doc.group.list_email, - ), - cc=cc, - ) - - -def email_wg_last_call_issued(request, doc, wglc_duration_weeks=None): - if wglc_duration_weeks is None: - wglc_duration_weeks = 2 - (to, cc) = gather_address_lists("doc_wg_last_call_issued", doc=doc) - frm = request.user.person.formatted_email() - - - end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=7 * wglc_duration_weeks) - subject = f"WG Last Call: {doc.name}-{doc.rev} (Ends {end_date})" - - send_mail( - request, - to, - frm, - subject, - "doc/mail/wg_last_call_issued.txt", - dict( - doc=doc, - subject=subject, - url=settings.IDTRACKER_BASE_URL + doc.get_absolute_url(), - end_date=end_date, - wglc_duration_weeks=wglc_duration_weeks, - wg_list=doc.group.list_email, - ), - cc=cc, - ) def email_pulled_from_rfc_queue(request, doc, comment, prev_state, next_state): extra=extra_automation_headers(doc) diff --git a/ietf/doc/management/commands/reset_rfc_authors.py b/ietf/doc/management/commands/reset_rfc_authors.py deleted file mode 100644 index e2ab5f1208..0000000000 --- a/ietf/doc/management/commands/reset_rfc_authors.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright The IETF Trust 2024, All Rights Reserved - -# Reset an RFC's authors to those of the draft it came from -from django.core.management.base import BaseCommand, CommandError - -from ietf.doc.models import Document, DocEvent -from ietf.person.models import Person - - -class Command(BaseCommand): - def add_arguments(self, parser): - parser.add_argument("rfcnum", type=int, help="RFC number to modify") - parser.add_argument( - "--force", - action="store_true", - help="reset even if RFC already has authors", - ) - - def handle(self, *args, **options): - try: - rfc = Document.objects.get(type="rfc", rfc_number=options["rfcnum"]) - except Document.DoesNotExist: - raise CommandError( - f"rfc{options['rfcnum']} does not exist in the Datatracker." - ) - - draft = rfc.came_from_draft() - if draft is None: - raise CommandError(f"{rfc.name} did not come from a draft. Can't reset.") - - orig_authors = rfc.documentauthor_set.all() - if orig_authors.exists(): - # Potentially dangerous, so refuse unless "--force" is specified - if not options["force"]: - raise CommandError( - f"{rfc.name} already has authors. Not resetting. Use '--force' to reset anyway." - ) - removed_auth_names = list(orig_authors.values_list("person__name", flat=True)) - rfc.documentauthor_set.all().delete() - DocEvent.objects.create( - doc=rfc, - by=Person.objects.get(name="(System)"), - type="edited_authors", - desc=f"Removed all authors: {', '.join(removed_auth_names)}", - ) - self.stdout.write( - self.style.SUCCESS( - f"Removed author(s): {', '.join(removed_auth_names)}" - ) - ) - - for author in draft.documentauthor_set.all(): - # Copy the author but point at the new doc. - # See https://docs.djangoproject.com/en/4.2/topics/db/queries/#copying-model-instances - author.pk = None - author.id = None - author._state.adding = True - author.document = rfc - author.save() - self.stdout.write( - self.style.SUCCESS(f"Added author {author.person.name} <{author.email}>") - ) - auth_names = draft.documentauthor_set.values_list("person__name", flat=True) - DocEvent.objects.create( - doc=rfc, - by=Person.objects.get(name="(System)"), - type="edited_authors", - desc=f"Set authors from rev {draft.rev} of {draft.name}: {', '.join(auth_names)}", - ) diff --git a/ietf/doc/management/commands/tests.py b/ietf/doc/management/commands/tests.py deleted file mode 100644 index 8244d87266..0000000000 --- a/ietf/doc/management/commands/tests.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright The IETF Trust 2024, All Rights Reserved -# -*- coding: utf-8 -*- - -from io import StringIO - -from django.core.management import call_command, CommandError - -from ietf.doc.factories import DocumentAuthorFactory, WgDraftFactory, WgRfcFactory -from ietf.doc.models import Document, DocumentAuthor -from ietf.utils.test_utils import TestCase - - -class CommandTests(TestCase): - @staticmethod - def _call_command(command_name, *args, **options): - """Call command, capturing (and suppressing) output""" - out = StringIO() - err = StringIO() - options["stdout"] = out - options["stderr"] = err - call_command(command_name, *args, **options) - return out.getvalue(), err.getvalue() - - def test_reset_rfc_authors(self): - command_name = "reset_rfc_authors" - - draft = WgDraftFactory() - DocumentAuthorFactory.create_batch(3, document=draft) - rfc = WgRfcFactory() # rfc does not yet have a draft - DocumentAuthorFactory.create_batch(3, document=rfc) - bad_rfc_num = ( - 1 - + Document.objects.filter(rfc_number__isnull=False) - .order_by("-rfc_number") - .first() - .rfc_number - ) - docauthor_fields = [ - field.name - for field in DocumentAuthor._meta.get_fields() - if field.name not in ["document", "id"] - ] - - with self.assertRaises(CommandError, msg="Cannot reset a bad RFC number"): - self._call_command(command_name, bad_rfc_num) - - with self.assertRaises(CommandError, msg="Cannot reset an RFC with no draft"): - self._call_command(command_name, rfc.rfc_number) - - with self.assertRaises(CommandError, msg="Cannot force-reset an RFC with no draft"): - self._call_command(command_name, rfc.rfc_number, "--force") - - # Link the draft to the rfc - rfc.targets_related.create(relationship_id="became_rfc", source=draft) - - with self.assertRaises(CommandError, msg="Cannot reset an RFC with authors"): - self._call_command(command_name, rfc.rfc_number) - - # Calling with force should work - self._call_command(command_name, rfc.rfc_number, "--force") - self.assertCountEqual( - draft.documentauthor_set.values(*docauthor_fields), - rfc.documentauthor_set.values(*docauthor_fields), - ) - - # Calling on an RFC with no authors should also work - rfc.documentauthor_set.all().delete() - self._call_command(command_name, rfc.rfc_number) - self.assertCountEqual( - draft.documentauthor_set.values(*docauthor_fields), - rfc.documentauthor_set.values(*docauthor_fields), - ) diff --git a/ietf/doc/migrations/0026_change_wg_state_descriptions.py b/ietf/doc/migrations/0026_change_wg_state_descriptions.py new file mode 100644 index 0000000000..b02b12c97e --- /dev/null +++ b/ietf/doc/migrations/0026_change_wg_state_descriptions.py @@ -0,0 +1,117 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations + +def forward(apps, schema_editor): + State = apps.get_model("doc","State") + for name, desc in [ + ("WG Document","The document has been adopted by the Working Group (WG) and is under development. A document can only be adopted by one WG at a time. However, a document may be transferred between WGs."), + ("Parked WG Document","The Working Group (WG) document is in a temporary state where it will not be actively developed. The reason for the pause is explained via a datatracker comments section."), + ("Dead WG Document","The Working Group (WG) document has been abandoned by the WG. No further development is planned in this WG. A decision to resume work on this document and move it out of this state is possible."), + ("In WG Last Call","The Working Group (WG) document is currently subject to an active WG Last Call (WGLC) review per Section 7.4 of RFC2418."), + ("Waiting for Implementation","The progression of this Working Group (WG) document towards publication is paused as it awaits implementation. The process governing the approach to implementations is WG-specific."), + ("Held by WG","Held by Working Group (WG) chairs for administrative reasons. See document history for details."), + ("Waiting for WG Chair Go-Ahead","The Working Group (WG) document has completed Working Group Last Call (WGLC), but the WG chair(s) are not yet ready to call consensus on the document. The reasons for this may include comments from the WGLC need to be responded to, or a revision to the document is needed"), + ("WG Consensus: Waiting for Write-Up","The Working Group (WG) document has consensus to proceed to publication. However, the document is waiting for a document shepherd write-up per RFC4858."), + ("Submitted to IESG for Publication","The Working Group (WG) document has left the WG and been submitted to the Internet Engineering Steering Group (IESG) for evaluation and publication. See the “IESG State” or “RFC Editor State” for further details on the state of the document."), + ("Candidate for WG Adoption","The individual submission document has been marked by the Working Group (WG) chairs as a candidate for adoption by the WG, but no adoption call has been started."), + ("Call For Adoption By WG Issued","A call for adoption of the individual submission document has been issued by the Working Group (WG) chairs. This call is still running but the WG has not yet reached consensus for adoption."), + ("Adopted by a WG","The individual submission document has been adopted by the Working Group (WG), but a WG document replacing this document with the typical naming convention of 'draft- ietf-wgname-topic-nn' has not yet been submitted."), + ("Adopted for WG Info Only","The document is adopted by the Working Group (WG) for its internal use. The WG has decided that it will not pursue publication of it as an RFC."), + ]: + State.objects.filter(name=name).update(desc=desc) + +def reverse(apps, schema_editor): + State = apps.get_model("doc","State") + for name, desc in [ + ("WG Document","""4.2.4. WG Document + + The "WG Document" state describes an I-D that has been adopted by an IETF WG and is being actively developed. + + A WG Chair may transition an I-D into the "WG Document" state at any time as long as the I-D is not being considered or developed in any other WG. + + Alternatively, WG Chairs may rely upon new functionality to be added to the Datatracker to automatically move version-00 drafts into the "WG Document" state as described in Section 4.1. + + Under normal conditions, it should not be possible for an I-D to be in the "WG Document" state in more than one WG at a time. This said, I-Ds may be transferred from one WG to another with the consent of the WG Chairs and the responsible ADs."""), + ("Parked WG Document","""4.2.5. Parked WG Document + + A "Parked WG Document" is an I-D that has lost its author or editor, is waiting for another document to be written or for a review to be completed, or cannot be progressed by the working group for some other reason. + + Some of the annotation tags described in Section 4.3 may be used in conjunction with this state to indicate why an I-D has been parked, and/or what may need to happen for the I-D to be un-parked. + + Parking a WG draft will not prevent it from expiring; however, this state can be used to indicate why the I-D has stopped progressing in the WG. + + A "Parked WG Document" that is not expired may be transferred from one WG to another with the consent of the WG Chairs and the responsible ADs."""), + ("Dead WG Document","""4.2.6. Dead WG Document + + A "Dead WG Document" is an I-D that has been abandoned. Note that 'Dead' is not always a final state for a WG I-D. If consensus is subsequently achieved, a "Dead WG Document" may be resurrected. A "Dead WG Document" that is not resurrected will eventually expire. + + Note that an I-D that is declared to be "Dead" in one WG and that is not expired may be transferred to a non-dead state in another WG with the consent of the WG Chairs and the responsible ADs."""), + ("In WG Last Call","""4.2.7. In WG Last Call + + A document "In WG Last Call" is an I-D for which a WG Last Call (WGLC) has been issued and is in progress. + + Note that conducting a WGLC is an optional part of the IETF WG process, per Section 7.4 of RFC 2418 [RFC2418]. + + If a WG Chair decides to conduct a WGLC on an I-D, the "In WG Last Call" state can be used to track the progress of the WGLC. The Chair may configure the Datatracker to send a WGLC message to one or more mailing lists when the Chair moves the I-D into this state. The WG Chair may also be able to select a different set of mailing lists for a different document undergoing a WGLC; some documents may deserve coordination with other WGs. + + A WG I-D in this state should remain "In WG Last Call" until the WG Chair moves it to another state. The WG Chair may configure the Datatracker to send an e-mail after a specified period of time to remind or 'nudge' the Chair to conclude the WGLC and to determine the next state for the document. + + It is possible for one WGLC to lead into another WGLC for the same document. For example, an I-D that completed a WGLC as an "Informational" document may need another WGLC if a decision is taken to convert the I-D into a Standards Track document."""), + ("Waiting for Implementation","""In some areas, it can be desirable to wait for multiple interoperable implementations before progressing a draft to be an RFC, and in some WGs this is required. This state should be entered after WG Last Call has completed."""), + ("Held by WG","""Held by WG, see document history for details."""), + ("Waiting for WG Chair Go-Ahead","""4.2.8. Waiting for WG Chair Go-Ahead + + A WG Chair may wish to place an I-D that receives a lot of comments during a WGLC into the "Waiting for WG Chair Go-Ahead" state. This state describes an I-D that has undergone a WGLC; however, the Chair is not yet ready to call consensus on the document. + + If comments from the WGLC need to be responded to, or a revision to the I-D is needed, the Chair may place an I-D into this state until all of the WGLC comments are adequately addressed and the (possibly revised) document is in the I-D repository."""), + ("WG Consensus: Waiting for Write-Up","""4.2.9. WG Consensus: Waiting for Writeup + + A document in the "WG Consensus: Waiting for Writeup" state has essentially completed its development within the working group, and is nearly ready to be sent to the IESG for publication. The last thing to be done is the preparation of a protocol writeup by a Document Shepherd. The IESG requires that a document shepherd writeup be completed before publication of the I-D is requested. The IETF document shepherding process and the role of a WG Document Shepherd is described in RFC 4858 [RFC4858] + + A WG Chair may call consensus on an I-D without a formal WGLC and transition an I-D that was in the "WG Document" state directly into this state. + + The name of this state includes the words "Waiting for Writeup" because a good document shepherd writeup takes time to prepare."""), + ("Submitted to IESG for Publication","""4.2.10. Submitted to IESG for Publication + + This state describes a WG document that has been submitted to the IESG for publication and that has not been sent back to the working group for revision. + + An I-D in this state may be under review by the IESG, it may have been approved and be in the RFC Editor's queue, or it may have been published as an RFC. Other possibilities exist too. The document may be "Dead" (in the IESG state machine) or in a "Do Not Publish" state."""), + ("Candidate for WG Adoption","""The document has been marked as a candidate for WG adoption by the WG Chair. This state can be used before a call for adoption is issued (and the document is put in the "Call For Adoption By WG Issued" state), to indicate that the document is in the queue for a call for adoption, even if none has been issued yet."""), + ("Call For Adoption By WG Issued","""4.2.1. Call for Adoption by WG Issued + + The "Call for Adoption by WG Issued" state should be used to indicate when an I-D is being considered for adoption by an IETF WG. An I-D that is in this state is actively being considered for adoption and has not yet achieved consensus, preference, or selection in the WG. + + This state may be used to describe an I-D that someone has asked a WG to consider for adoption, if the WG Chair has agreed with the request. This state may also be used to identify an I-D that a WG Chair asked an author to write specifically for consideration as a candidate WG item [WGDTSPEC], and/or an I-D that is listed as a 'candidate draft' in the WG's charter. + + Under normal conditions, it should not be possible for an I-D to be in the "Call for Adoption by WG Issued" state in more than one working group at the same time. This said, it is not uncommon for authors to "shop" their I-Ds to more than one WG at a time, with the hope of getting their documents adopted somewhere. + + After this state is implemented in the Datatracker, an I-D that is in the "Call for Adoption by WG Issued" state will not be able to be "shopped" to any other WG without the consent of the WG Chairs and the responsible ADs impacted by the shopping. + + Note that Figure 1 includes an arc leading from this state to outside of the WG state machine. This illustrates that some I-Ds that are considered do not get adopted as WG drafts. An I-D that is not adopted as a WG draft will transition out of the WG state machine and revert back to having no stream-specific state; however, the status change history log of the I-D will record that the I-D was previously in the "Call for Adoption by WG Issued" state."""), + ("Adopted by a WG","""4.2.2. Adopted by a WG + + The "Adopted by a WG" state describes an individual submission I-D that an IETF WG has agreed to adopt as one of its WG drafts. + + WG Chairs who use this state will be able to clearly indicate when their WGs adopt individual submission I-Ds. This will facilitate the Datatracker's ability to correctly capture "Replaces" information for WG drafts and correct "Replaced by" information for individual submission I-Ds that have been replaced by WG drafts. + + This state is needed because the Datatracker uses the filename of an I-D as a key to search its database for status information about the I-D, and because the filename of a WG I-D is supposed to be different from the filename of an individual submission I-D. The filename of an individual submission I-D will typically be formatted as 'draft-author-wgname-topic-nn'. + + The filename of a WG document is supposed to be formatted as 'draft- ietf-wgname-topic-nn'. + + An individual I-D that is adopted by a WG may take weeks or months to be resubmitted by the author as a new (version-00) WG draft. If the "Adopted by a WG" state is not used, the Datatracker has no way to determine that an I-D has been adopted until a new version of the I-D is submitted to the WG by the author and until the I-D is approved for posting by a WG Chair."""), + ("Adopted for WG Info Only","""4.2.3. Adopted for WG Info Only + + The "Adopted for WG Info Only" state describes a document that contains useful information for the WG that adopted it, but the document is not intended to be published as an RFC. The WG will not actively develop the contents of the I-D or progress it for publication as an RFC. The only purpose of the I-D is to provide information for internal use by the WG."""), + ]: + State.objects.filter(name=name).update(desc=desc) + +class Migration(migrations.Migration): + + dependencies = [ + ("doc", "0025_storedobject_storedobject_unique_name_per_store"), + ] + + operations = [ + migrations.RunPython(forward, reverse) + ] diff --git a/ietf/doc/migrations/0027_alter_dochistory_title_alter_document_title.py b/ietf/doc/migrations/0027_alter_dochistory_title_alter_document_title.py new file mode 100644 index 0000000000..e0d8560e6f --- /dev/null +++ b/ietf/doc/migrations/0027_alter_dochistory_title_alter_document_title.py @@ -0,0 +1,41 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0026_change_wg_state_descriptions"), + ] + + operations = [ + migrations.AlterField( + model_name="dochistory", + name="title", + field=models.CharField( + max_length=255, + validators=[ + django.core.validators.ProhibitNullCharactersValidator, # type:ignore + django.core.validators.RegexValidator( + message="Please enter a string without control characters.", + regex="^[^\x01-\x1f]*$", + ), + ], + ), + ), + migrations.AlterField( + model_name="document", + name="title", + field=models.CharField( + max_length=255, + validators=[ + django.core.validators.ProhibitNullCharactersValidator, # type:ignore + django.core.validators.RegexValidator( + message="Please enter a string without control characters.", + regex="^[^\x01-\x1f]*$", + ), + ], + ), + ), + ] diff --git a/ietf/doc/migrations/0028_rfcauthor.py b/ietf/doc/migrations/0028_rfcauthor.py new file mode 100644 index 0000000000..776dc22eb1 --- /dev/null +++ b/ietf/doc/migrations/0028_rfcauthor.py @@ -0,0 +1,84 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations, models +import django.db.models.deletion +import ietf.utils.models + + +class Migration(migrations.Migration): + dependencies = [ + ("person", "0005_alter_historicalperson_pronouns_selectable_and_more"), + ("doc", "0027_alter_dochistory_title_alter_document_title"), + ] + + operations = [ + migrations.CreateModel( + name="RfcAuthor", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("titlepage_name", models.CharField(max_length=128)), + ("is_editor", models.BooleanField(default=False)), + ( + "affiliation", + models.CharField( + blank=True, + help_text="Organization/company used by author for submission", + max_length=100, + ), + ), + ( + "country", + models.CharField( + blank=True, + help_text="Country used by author for submission", + max_length=255, + ), + ), + ("order", models.IntegerField(default=1)), + ( + "document", + ietf.utils.models.ForeignKey( + limit_choices_to={"type_id": "rfc"}, + on_delete=django.db.models.deletion.CASCADE, + to="doc.document", + ), + ), + ( + "email", + ietf.utils.models.ForeignKey( + blank=True, + help_text="Email address used by author for submission", + null=True, + on_delete=django.db.models.deletion.PROTECT, + to="person.email", + ), + ), + ( + "person", + ietf.utils.models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.PROTECT, + to="person.person", + ), + ), + ], + options={ + "ordering": ["document", "order"], + "indexes": [ + models.Index( + fields=["document", "order"], + name="doc_rfcauth_documen_6b5dc4_idx", + ) + ], + }, + ), + ] diff --git a/ietf/doc/migrations/0029_editedrfcauthorsdocevent.py b/ietf/doc/migrations/0029_editedrfcauthorsdocevent.py new file mode 100644 index 0000000000..60837c5cb2 --- /dev/null +++ b/ietf/doc/migrations/0029_editedrfcauthorsdocevent.py @@ -0,0 +1,30 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0028_rfcauthor"), + ] + + operations = [ + migrations.CreateModel( + name="EditedRfcAuthorsDocEvent", + fields=[ + ( + "docevent_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="doc.docevent", + ), + ), + ], + bases=("doc.docevent",), + ), + ] diff --git a/ietf/doc/migrations/0030_alter_dochistory_title_alter_document_title.py b/ietf/doc/migrations/0030_alter_dochistory_title_alter_document_title.py new file mode 100644 index 0000000000..9ee858b2e8 --- /dev/null +++ b/ietf/doc/migrations/0030_alter_dochistory_title_alter_document_title.py @@ -0,0 +1,41 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0029_editedrfcauthorsdocevent"), + ] + + operations = [ + migrations.AlterField( + model_name="dochistory", + name="title", + field=models.CharField( + max_length=255, + validators=[ + django.core.validators.ProhibitNullCharactersValidator(), + django.core.validators.RegexValidator( + message="Please enter a string without control characters.", + regex="^[^\x01-\x1f]*$", + ), + ], + ), + ), + migrations.AlterField( + model_name="document", + name="title", + field=models.CharField( + max_length=255, + validators=[ + django.core.validators.ProhibitNullCharactersValidator(), + django.core.validators.RegexValidator( + message="Please enter a string without control characters.", + regex="^[^\x01-\x1f]*$", + ), + ], + ), + ), + ] diff --git a/ietf/doc/migrations/0031_change_draft_stream_ietf_state_descriptions.py b/ietf/doc/migrations/0031_change_draft_stream_ietf_state_descriptions.py new file mode 100644 index 0000000000..c664126da3 --- /dev/null +++ b/ietf/doc/migrations/0031_change_draft_stream_ietf_state_descriptions.py @@ -0,0 +1,57 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations + + +def forward(apps, schema_editor): + State = apps.get_model("doc", "State") + for name, desc in [ + ( + "Adopted by a WG", + "The individual submission document has been adopted by the Working Group (WG), but some administrative matter still needs to be completed (e.g., a WG document replacing this document with the typical naming convention of 'draft-ietf-wgname-topic-nn' has not yet been submitted).", + ), + ( + "WG Document", + "The document has been identified as a Working Group (WG) document and is under development per Section 7.2 of RFC2418.", + ), + ( + "Waiting for WG Chair Go-Ahead", + "The Working Group (WG) document has completed Working Group Last Call (WGLC), but the WG chairs are not yet ready to call consensus on the document. The reasons for this may include comments from the WGLC need to be responded to, or a revision to the document is needed.", + ), + ( + "Submitted to IESG for Publication", + "The Working Group (WG) document has been submitted to the Internet Engineering Steering Group (IESG) for evaluation and publication per Section 7.4 of RFC2418. See the “IESG State” or “RFC Editor State” for further details on the state of the document.", + ), + ]: + State.objects.filter(name=name).update(desc=desc, type="draft-stream-ietf") + + +def reverse(apps, schema_editor): + State = apps.get_model("doc", "State") + for name, desc in [ + ( + "Adopted by a WG", + "The individual submission document has been adopted by the Working Group (WG), but a WG document replacing this document with the typical naming convention of 'draft- ietf-wgname-topic-nn' has not yet been submitted.", + ), + ( + "WG Document", + "The document has been adopted by the Working Group (WG) and is under development. A document can only be adopted by one WG at a time. However, a document may be transferred between WGs.", + ), + ( + "Waiting for WG Chair Go-Ahead", + "The Working Group (WG) document has completed Working Group Last Call (WGLC), but the WG chair(s) are not yet ready to call consensus on the document. The reasons for this may include comments from the WGLC need to be responded to, or a revision to the document is needed", + ), + ( + "Submitted to IESG for Publication", + "The Working Group (WG) document has left the WG and been submitted to the Internet Engineering Steering Group (IESG) for evaluation and publication. See the “IESG State” or “RFC Editor State” for further details on the state of the document.", + ), + ]: + State.objects.filter(name=name).update(desc=desc, type="draft-stream-ietf") + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0030_alter_dochistory_title_alter_document_title"), + ] + + operations = [migrations.RunPython(forward, reverse)] diff --git a/ietf/doc/migrations/0032_remove_rfcauthor_email.py b/ietf/doc/migrations/0032_remove_rfcauthor_email.py new file mode 100644 index 0000000000..a0e147da59 --- /dev/null +++ b/ietf/doc/migrations/0032_remove_rfcauthor_email.py @@ -0,0 +1,16 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0031_change_draft_stream_ietf_state_descriptions"), + ] + + operations = [ + migrations.RemoveField( + model_name="rfcauthor", + name="email", + ), + ] diff --git a/ietf/doc/migrations/0033_dochistory_keywords_document_keywords.py b/ietf/doc/migrations/0033_dochistory_keywords_document_keywords.py new file mode 100644 index 0000000000..5e2513e15a --- /dev/null +++ b/ietf/doc/migrations/0033_dochistory_keywords_document_keywords.py @@ -0,0 +1,31 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from django.db import migrations, models +import ietf.doc.models + + +class Migration(migrations.Migration): + dependencies = [ + ("doc", "0032_remove_rfcauthor_email"), + ] + + operations = [ + migrations.AddField( + model_name="dochistory", + name="keywords", + field=models.JSONField( + default=list, + max_length=1000, + validators=[ietf.doc.models.validate_doc_keywords], + ), + ), + migrations.AddField( + model_name="document", + name="keywords", + field=models.JSONField( + default=list, + max_length=1000, + validators=[ietf.doc.models.validate_doc_keywords], + ), + ), + ] diff --git a/ietf/doc/models.py b/ietf/doc/models.py index b6f36cb8a7..7b23a62c45 100644 --- a/ietf/doc/models.py +++ b/ietf/doc/models.py @@ -1,7 +1,8 @@ -# Copyright The IETF Trust 2010-2025, All Rights Reserved +# Copyright The IETF Trust 2010-2026, All Rights Reserved # -*- coding: utf-8 -*- +from collections import namedtuple import datetime import logging import os @@ -11,6 +12,9 @@ from io import BufferedReader from pathlib import Path + +from django.core.exceptions import ValidationError +from django.db.models import Q from lxml import etree from typing import Optional, Protocol, TYPE_CHECKING, Union from weasyprint import HTML as wpHTML @@ -20,7 +24,11 @@ from django.core import checks from django.core.files.base import File from django.core.cache import caches -from django.core.validators import URLValidator, RegexValidator +from django.core.validators import ( + URLValidator, + RegexValidator, + ProhibitNullCharactersValidator, +) from django.urls import reverse as urlreverse from django.contrib.contenttypes.models import ContentType from django.conf import settings @@ -102,12 +110,27 @@ class Meta: IESG_STATCHG_CONFLREV_ACTIVE_STATES = ("iesgeval", "defer") IESG_SUBSTATE_TAGS = ('ad-f-up', 'need-rev', 'extpty') + +def validate_doc_keywords(value): + if ( + not isinstance(value, list | tuple | set) + or not all(isinstance(elt, str) for elt in value) + ): + raise ValidationError("Value must be an array of strings") + + class DocumentInfo(models.Model): """Any kind of document. Draft, RFC, Charter, IPR Statement, Liaison Statement""" time = models.DateTimeField(default=timezone.now) # should probably have auto_now=True type = ForeignKey(DocTypeName, blank=True, null=True) # Draft, Agenda, Minutes, Charter, Discuss, Guideline, Email, Review, Issue, Wiki, External ... - title = models.CharField(max_length=255, validators=[validate_no_control_chars, ]) + title = models.CharField( + max_length=255, + validators=[ + ProhibitNullCharactersValidator(), + validate_no_control_chars, + ], + ) states = models.ManyToManyField(State, blank=True) # plain state (Active/Expired/...), IESG state, stream state tags = models.ManyToManyField(DocTagName, blank=True) # Revised ID Needed, ExternalParty, AD Followup, ... @@ -129,6 +152,17 @@ class DocumentInfo(models.Model): uploaded_filename = models.TextField(blank=True) note = models.TextField(blank=True) rfc_number = models.PositiveIntegerField(blank=True, null=True) # only valid for type="rfc" + keywords = models.JSONField( + default=list, + max_length=1000, + validators=[validate_doc_keywords], + ) + + @property + def doi(self) -> str | None: + if self.type_id == "rfc" and self.rfc_number is not None: + return f"{settings.IETF_DOI_PREFIX}/RFC{self.rfc_number:04d}" + return None def file_extension(self): if not hasattr(self, '_cached_extension'): @@ -228,14 +262,14 @@ def revisions_by_newrevisionevent(self): return revisions def get_href(self, meeting=None): - return self._get_ref(meeting=meeting,meeting_doc_refs=settings.MEETING_DOC_HREFS) + return self._get_ref(meeting=meeting, versioned=True) def get_versionless_href(self, meeting=None): - return self._get_ref(meeting=meeting,meeting_doc_refs=settings.MEETING_DOC_GREFS) + return self._get_ref(meeting=meeting, versioned=False) - def _get_ref(self, meeting=None, meeting_doc_refs=settings.MEETING_DOC_HREFS): + def _get_ref(self, meeting=None, versioned=True): """ Returns an url to the document text. This differs from .get_absolute_url(), which returns an url to the datatracker page for the document. @@ -244,12 +278,16 @@ def _get_ref(self, meeting=None, meeting_doc_refs=settings.MEETING_DOC_HREFS): # the earlier resolution order, but there's at the moment one single # instance which matches this (with correct results), so we won't # break things all over the place. - if not hasattr(self, '_cached_href'): + cache_attr = "_cached_href" if versioned else "_cached_versionless_href" + if not hasattr(self, cache_attr): validator = URLValidator() if self.external_url and self.external_url.split(':')[0] in validator.schemes: validator(self.external_url) return self.external_url + meeting_doc_refs = ( + settings.MEETING_DOC_HREFS if versioned else settings.MEETING_DOC_GREFS + ) if self.type_id in settings.DOC_HREFS and self.type_id in meeting_doc_refs: if self.meeting_related(): self.is_meeting_related = True @@ -301,8 +339,8 @@ def _get_ref(self, meeting=None, meeting_doc_refs=settings.MEETING_DOC_HREFS): if href.startswith('/'): href = settings.IDTRACKER_BASE_URL + href - self._cached_href = href - return self._cached_href + setattr(self, cache_attr, href) + return getattr(self, cache_attr) def set_state(self, state): """Switch state type implicit in state to state. This just @@ -407,9 +445,56 @@ def friendly_state(self): else: return state.name + def author_names(self): + """Author names as a list of strings""" + names = [] + if self.type_id == "rfc" and self.rfcauthor_set.exists(): + for author in self.rfcauthor_set.select_related("person"): + if author.person: + names.append(author.person.name) + else: + # titlepage_name cannot be blank + names.append(author.titlepage_name) + else: + names = [ + author.person.name + for author in self.documentauthor_set.select_related("person") + ] + return names + + def author_persons_or_names(self): + """Authors as a list of named tuples with person and/or titlepage_name""" + Author = namedtuple("Author", "person titlepage_name") + persons_or_names = [] + if self.type_id=="rfc" and self.rfcauthor_set.exists(): + for author in self.rfcauthor_set.select_related("person"): + persons_or_names.append(Author(person=author.person, titlepage_name=author.titlepage_name)) + else: + for author in self.documentauthor_set.select_related("person"): + persons_or_names.append(Author(person=author.person, titlepage_name="")) + return persons_or_names + + def author_persons(self): + """Authors as a list of Persons + + Omits any RfcAuthors with a null person field. + """ + if self.type_id == "rfc" and self.rfcauthor_set.exists(): + authors_qs = self.rfcauthor_set.filter(person__isnull=False) + else: + authors_qs = self.documentauthor_set.all() + return [a.person for a in authors_qs.select_related("person")] + def author_list(self): + """List of author emails""" + if self.type_id == "rfc" and self.rfcauthor_set.exists(): + author_qs = self.rfcauthor_set.select_related("person").order_by("order") + else: + author_qs = self.documentauthor_set.select_related("email").order_by( + "order" + ) best_addresses = [] - for author in self.documentauthor_set.all(): + for author in author_qs: if author.email: if author.email.active or not author.email.person: best_addresses.append(author.email.address) @@ -417,9 +502,6 @@ def author_list(self): best_addresses.append(author.email.person.email_address()) return ", ".join(best_addresses) - def authors(self): - return [ a.person for a in self.documentauthor_set.all() ] - # This, and several other ballot related functions here, assume that there is only one active ballot for a document at any point in time. # If that assumption is violated, they will only expose the most recently created ballot def ballot_open(self, ballot_type_slug): @@ -721,7 +803,14 @@ def referenced_by_rfcs_as_rfc_or_draft(self): if self.type_id == "rfc" and self.came_from_draft(): refs_to |= self.came_from_draft().referenced_by_rfcs() return refs_to - + + def sent_to_rfc_editor_event(self): + if self.stream_id == "ietf": + return self.docevent_set.filter(type="iesg_approved").order_by("-time").first() + elif self.stream_id in ["editorial", "iab", "irtf", "ise"]: + return self.docevent_set.filter(type="requested_publication").order_by("-time").first() + else: + return None class Meta: abstract = True @@ -845,6 +934,54 @@ def is_approved_downref(self): return False +class RfcAuthor(models.Model): + """Captures the authors of an RFC as represented on the RFC title page. + + This deviates from DocumentAuthor in that it does not get moved into the DocHistory + hierarchy as documents are saved. It will attempt to preserve email, country, and affiliation + from the DocumentAuthor objects associated with the draft leading to this RFC (which + may be wrong if the author moves or changes affiliation while the document is in the + queue). + + It does not, at this time, attempt to capture the authors from anything _but_ the title + page. The datatracker may know more about such authors based on information from the draft + leading to the RFC, and future work may take that into account. + + Once doc.rfcauthor_set.exists() for a doc of type `rfc`, doc.documentauthor_set should be + ignored. + """ + + document = ForeignKey( + "Document", + on_delete=models.CASCADE, + limit_choices_to={"type_id": "rfc"}, # only affects ModelForms (e.g., admin) + ) + titlepage_name = models.CharField(max_length=128, blank=False) + is_editor = models.BooleanField(default=False) + person = ForeignKey(Person, null=True, blank=True, on_delete=models.PROTECT) + affiliation = models.CharField(max_length=100, blank=True, help_text="Organization/company used by author for submission") + country = models.CharField(max_length=255, blank=True, help_text="Country used by author for submission") + order = models.IntegerField(default=1) + + def __str__(self): + return u"%s %s (%s)" % (self.document.name, self.person, self.order) + + class Meta: + ordering=["document", "order"] + indexes=[ + models.Index(fields=["document", "order"]) + ] + + @property + def email(self) -> Email | None: + return self.person.email() if self.person else None + + def format_for_titlepage(self): + if self.is_editor: + return f"{self.titlepage_name}, Ed." + return self.titlepage_name + + class DocumentAuthorInfo(models.Model): person = ForeignKey(Person) # email should only be null for some historic documents @@ -894,7 +1031,7 @@ class Meta: def role_for_doc(self): """Brief string description of this person's relationship to the doc""" roles = [] - if self.person in self.document.authors(): + if self.person in self.document.author_persons(): roles.append('Author') if self.person == self.document.ad: roles.append('Responsible AD') @@ -913,13 +1050,25 @@ def role_for_doc(self): roles.append('Action Holder') return ', '.join(roles) +# N.B., at least a couple dozen documents exist that do not satisfy this validator validate_docname = RegexValidator( r'^[-a-z0-9]+$', "Provide a valid document name consisting of lowercase letters, numbers and hyphens.", 'invalid' ) + +SUBSERIES_DOC_TYPE_IDS = ("bcp", "fyi", "std") + + +class DocumentQuerySet(models.QuerySet): + def subseries_docs(self): + return self.filter(type_id__in=SUBSERIES_DOC_TYPE_IDS) + + class Document(StorableMixin, DocumentInfo): + objects = DocumentQuerySet.as_manager() + name = models.CharField(max_length=255, validators=[validate_docname,], unique=True) # immutable action_holders = models.ManyToManyField(Person, through=DocumentActionHolder, blank=True) @@ -1025,6 +1174,22 @@ def request_closed_time(self, review_req): e = self.latest_event(ReviewRequestDocEvent, type="closed_review_request", review_request=review_req) return e.time if e and e.time else None + @property + def area(self) -> Group | None: + """Get area for document, if one exists + + None for non-IETF-stream documents. N.b., this is stricter than Group.area() and + uses different logic from Document.area_acronym(). + """ + if self.stream_id != "ietf": + return None + if self.group is None: + return None + parent = self.group.parent + if parent.type_id == "area": + return parent + return None + def area_acronym(self): g = self.group if g: @@ -1157,7 +1322,7 @@ def fake_history_obj(self, rev): elif rev_events.exists(): time = rev_events.first().time else: - time = datetime.datetime.fromtimestamp(0, datetime.timezone.utc) + time = datetime.datetime.fromtimestamp(0, datetime.UTC) dh = DocHistory(name=self.name, rev=rev, doc=self, time=time, type=self.type, title=self.title, stream=self.stream, group=self.group) @@ -1168,6 +1333,32 @@ def action_holders_enabled(self): iesg_state = self.get_state('draft-iesg') return iesg_state and iesg_state.slug != 'idexists' + def formats(self): + """List of file formats available + + Only implemented for RFCs. Relies on StoredObject. + """ + if self.type_id != "rfc": + raise RuntimeError("Only allowed for type=rfc") + + # StoredObject.doc_rev can be null or "" to represent no rev. Match either + # of these when self.rev is "" (always expected to be the case for RFCs) + rev_q = Q(doc_rev=self.rev) + if self.rev == "": + rev_q |= Q(doc_rev__isnull=True) + return [ + { + "fmt": Path(object_name).parts[0], + "name": object_name, + } + for object_name in StoredObject.objects.filter( + rev_q, + store="rfc", + doc_name=self.name, + ).values_list("name", flat=True) + ] + + class DocumentURL(models.Model): doc = ForeignKey(Document) tag = ForeignKey(DocUrlTagName) @@ -1580,6 +1771,11 @@ class EditedAuthorsDocEvent(DocEvent): """ basis = models.CharField(help_text="What is the source or reasoning for the changes to the author list",max_length=255) + +class EditedRfcAuthorsDocEvent(DocEvent): + """Change to the RfcAuthor list for a document""" + + class BofreqEditorDocEvent(DocEvent): """ Capture the proponents of a BOF Request.""" editors = models.ManyToManyField('person.Person', blank=True) @@ -1588,9 +1784,17 @@ class BofreqResponsibleDocEvent(DocEvent): """ Capture the responsible leadership (IAB and IESG members) for a BOF Request """ responsible = models.ManyToManyField('person.Person', blank=True) + +class StoredObjectQuerySet(models.QuerySet): + def exclude_deleted(self): + return self.filter(deleted__isnull=True) + + class StoredObject(models.Model): """Hold metadata about objects placed in object storage""" + objects = StoredObjectQuerySet.as_manager() + store = models.CharField(max_length=256) name = models.CharField(max_length=1024, null=False, blank=False) # N.B. the 1024 limit on name comes from S3 sha384 = models.CharField(max_length=96) diff --git a/ietf/doc/resources.py b/ietf/doc/resources.py index 157a3ad556..1d86df78d0 100644 --- a/ietf/doc/resources.py +++ b/ietf/doc/resources.py @@ -17,8 +17,9 @@ InitialReviewDocEvent, DocHistoryAuthor, BallotDocEvent, RelatedDocument, RelatedDocHistory, BallotPositionDocEvent, AddedMessageEvent, SubmissionDocEvent, ReviewRequestDocEvent, ReviewAssignmentDocEvent, EditedAuthorsDocEvent, DocumentURL, - IanaExpertDocEvent, IRSGBallotDocEvent, DocExtResource, DocumentActionHolder, - BofreqEditorDocEvent, BofreqResponsibleDocEvent, StoredObject) + IanaExpertDocEvent, IRSGBallotDocEvent, DocExtResource, DocumentActionHolder, + BofreqEditorDocEvent, BofreqResponsibleDocEvent, StoredObject, RfcAuthor, + EditedRfcAuthorsDocEvent) from ietf.name.resources import BallotPositionNameResource, DocTypeNameResource class BallotTypeResource(ModelResource): @@ -650,6 +651,31 @@ class Meta: api.doc.register(EditedAuthorsDocEventResource()) + +from ietf.person.resources import PersonResource +class EditedRfcAuthorsDocEventResource(ModelResource): + by = ToOneField(PersonResource, 'by') + doc = ToOneField(DocumentResource, 'doc') + docevent_ptr = ToOneField(DocEventResource, 'docevent_ptr') + class Meta: + queryset = EditedRfcAuthorsDocEvent.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'editedrfcauthorsdocevent' + ordering = ['id', ] + filtering = { + "id": ALL, + "time": ALL, + "type": ALL, + "rev": ALL, + "desc": ALL, + "by": ALL_WITH_RELATIONS, + "doc": ALL_WITH_RELATIONS, + "docevent_ptr": ALL_WITH_RELATIONS, + } +api.doc.register(EditedRfcAuthorsDocEventResource()) + + from ietf.name.resources import DocUrlTagNameResource class DocumentURLResource(ModelResource): doc = ToOneField(DocumentResource, 'doc') @@ -865,3 +891,28 @@ class Meta: "deleted": ALL, } api.doc.register(StoredObjectResource()) + + +from ietf.person.resources import EmailResource, PersonResource +class RfcAuthorResource(ModelResource): + document = ToOneField(DocumentResource, 'document') + person = ToOneField(PersonResource, 'person', null=True) + email = ToOneField(EmailResource, 'email', null=True, readonly=True) + class Meta: + queryset = RfcAuthor.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'rfcauthor' + ordering = ['id', ] + filtering = { + "id": ALL, + "titlepage_name": ALL, + "is_editor": ALL, + "affiliation": ALL, + "country": ALL, + "order": ALL, + "document": ALL_WITH_RELATIONS, + "person": ALL_WITH_RELATIONS, + "email": ALL_WITH_RELATIONS, + } +api.doc.register(RfcAuthorResource()) diff --git a/ietf/doc/serializers.py b/ietf/doc/serializers.py new file mode 100644 index 0000000000..3651670962 --- /dev/null +++ b/ietf/doc/serializers.py @@ -0,0 +1,360 @@ +# Copyright The IETF Trust 2024-2026, All Rights Reserved +"""django-rest-framework serializers""" + +from dataclasses import dataclass +from typing import Literal, ClassVar + +from django.db.models.manager import BaseManager +from django.db.models.query import QuerySet +from drf_spectacular.utils import extend_schema_field +from rest_framework import serializers + +from ietf.group.serializers import ( + AreaDirectorSerializer, + AreaSerializer, + GroupSerializer, +) +from ietf.name.serializers import StreamNameSerializer +from ietf.utils import log +from .models import Document, DocumentAuthor, RfcAuthor + + +class RfcAuthorSerializer(serializers.ModelSerializer): + """Serializer for an RfcAuthor / DocumentAuthor in a response""" + + email = serializers.EmailField(source="email.address", read_only=True) + datatracker_person_path = serializers.URLField( + source="person.get_absolute_url", + required=False, + help_text="URL for person link (relative to datatracker base URL)", + read_only=True, + ) + + class Meta: + model = RfcAuthor + fields = [ + "titlepage_name", + "is_editor", + "person", + "email", + "affiliation", + "country", + "datatracker_person_path", + ] + + def to_representation(self, instance): + """instance -> primitive data types + + Translates a DocumentAuthor into an equivalent RfcAuthor we can use the same + serializer for either type. + """ + if isinstance(instance, DocumentAuthor): + # create a non-persisted RfcAuthor as a shim - do not save it! + document_author = instance + instance = RfcAuthor( + titlepage_name=document_author.person.plain_name(), + is_editor=False, + person=document_author.person, + affiliation=document_author.affiliation, + country=document_author.country, + order=document_author.order, + ) + return super().to_representation(instance) + + def validate(self, data): + email = data.get("email") + if email is not None: + person = data.get("person") + if person is None: + raise serializers.ValidationError( + { + "email": "cannot have an email without a person", + }, + code="email-without-person", + ) + if email.person_id != person.pk: + raise serializers.ValidationError( + { + "email": "email must belong to person", + }, + code="email-person-mismatch", + ) + return data + + +@dataclass +class DocIdentifier: + type: Literal["doi", "issn"] + value: str + + +class DocIdentifierSerializer(serializers.Serializer): + type = serializers.ChoiceField(choices=["doi", "issn"]) + value = serializers.CharField() + + +type RfcStatusSlugT = Literal[ + "std", + "ps", + "ds", + "bcp", + "inf", + "exp", + "hist", + "unkn", + "not-issued", +] + + +@dataclass +class RfcStatus: + """Helper to extract the 'Status' from an RFC document for serialization""" + + slug: RfcStatusSlugT + + # Names that aren't just the slug itself. ClassVar annotation prevents dataclass from treating this as a field. + fancy_names: ClassVar[dict[RfcStatusSlugT, str]] = { + "std": "internet standard", + "ps": "proposed standard", + "ds": "draft standard", + "bcp": "best current practice", + "inf": "informational", + "exp": "experimental", + "hist": "historic", + "unkn": "unknown", + } + + # ClassVar annotation prevents dataclass from treating this as a field + stdlevelname_slug_map: ClassVar[dict[str, RfcStatusSlugT]] = { + "bcp": "bcp", + "ds": "ds", + "exp": "exp", + "hist": "hist", + "inf": "inf", + "std": "std", + "ps": "ps", + "unkn": "unkn", + } + + # ClassVar annotation prevents dataclass from treating this as a field + status_slugs: ClassVar[list[RfcStatusSlugT]] = sorted( + # TODO implement "not-issued" RFCs + set(stdlevelname_slug_map.values()) | {"not-issued"} + ) + + @property + def name(self): + return RfcStatus.fancy_names.get(self.slug, self.slug) + + @classmethod + def from_document(cls, doc: Document): + """Decide the status that applies to a document""" + return cls( + slug=(cls.stdlevelname_slug_map.get(doc.std_level.slug, "unkn")), + ) + + @classmethod + def filter(cls, queryset, name, value: list[RfcStatusSlugT]): + """Filter a queryset by status + + This is basically the inverse of the from_document() method. Given a status name, filter + the queryset to those in that status. The queryset should be a Document queryset. + """ + interesting_slugs = [ + stdlevelname_slug + for stdlevelname_slug, status_slug in cls.stdlevelname_slug_map.items() + if status_slug in value + ] + if len(interesting_slugs) == 0: + return queryset.none() + return queryset.filter(std_level__slug__in=interesting_slugs) + + +class RfcStatusSerializer(serializers.Serializer): + """Status serializer for a Document instance""" + + slug = serializers.ChoiceField(choices=RfcStatus.status_slugs) + name = serializers.CharField() + + def to_representation(self, instance: Document): + return super().to_representation(instance=RfcStatus.from_document(instance)) + + +class ShepherdSerializer(serializers.Serializer): + email = serializers.EmailField(source="email_address") + + +class RelatedDraftSerializer(serializers.Serializer): + id = serializers.IntegerField(source="source.id") + name = serializers.CharField(source="source.name") + title = serializers.CharField(source="source.title") + shepherd = ShepherdSerializer(source="source.shepherd", allow_null=True) + ad = AreaDirectorSerializer(source="source.ad", allow_null=True) + + +class RelatedRfcSerializer(serializers.Serializer): + id = serializers.IntegerField(source="target.id") + number = serializers.IntegerField(source="target.rfc_number") + title = serializers.CharField(source="target.title") + + +class ReverseRelatedRfcSerializer(serializers.Serializer): + id = serializers.IntegerField(source="source.id") + number = serializers.IntegerField(source="source.rfc_number") + title = serializers.CharField(source="source.title") + + +class ContainingSubseriesSerializer(serializers.Serializer): + name = serializers.CharField(source="source.name") + type = serializers.CharField(source="source.type_id") + + +class RfcFormatSerializer(serializers.Serializer): + RFC_FORMATS = ("xml", "txt", "html", "pdf", "ps", "json", "notprepped") + + fmt = serializers.ChoiceField(choices=RFC_FORMATS) + name = serializers.CharField(help_text="Name of blob in the blob store") + + +class RfcMetadataSerializer(serializers.ModelSerializer): + """Serialize metadata of an RFC + + This needs to be called with a Document queryset that has been processed with + api.augment_rfc_queryset() or it very likely will not work. Some of the typing + refers to Document, but this should really be WithAnnotations[Document, ...]. + However, have not been able to make that work yet. + """ + + number = serializers.IntegerField(source="rfc_number") + published = serializers.DateField() + status = RfcStatusSerializer(source="*") + authors = serializers.SerializerMethodField() + group = GroupSerializer() + area = AreaSerializer(read_only=True) + stream = StreamNameSerializer() + ad = AreaDirectorSerializer(read_only=True, allow_null=True) + group_list_email = serializers.EmailField(source="group.list_email", read_only=True) + identifiers = serializers.SerializerMethodField() + draft = serializers.SerializerMethodField() + obsoletes = RelatedRfcSerializer(many=True, read_only=True) + obsoleted_by = ReverseRelatedRfcSerializer(many=True, read_only=True) + updates = RelatedRfcSerializer(many=True, read_only=True) + updated_by = ReverseRelatedRfcSerializer(many=True, read_only=True) + subseries = ContainingSubseriesSerializer(many=True, read_only=True) + formats = RfcFormatSerializer( + many=True, read_only=True, help_text="Available formats" + ) + keywords = serializers.ListField(child=serializers.CharField(), read_only=True) + has_errata = serializers.BooleanField(read_only=True) + + class Meta: + model = Document + fields = [ + "number", + "title", + "published", + "status", + "pages", + "authors", + "group", + "area", + "stream", + "ad", + "group_list_email", + "identifiers", + "obsoletes", + "obsoleted_by", + "updates", + "updated_by", + "subseries", + "draft", + "abstract", + "formats", + "keywords", + "has_errata", + ] + + @extend_schema_field(RfcAuthorSerializer(many=True)) + def get_authors(self, doc: Document): + # If doc has any RfcAuthors, use those, otherwise fall back to DocumentAuthors + author_queryset: QuerySet[RfcAuthor] | QuerySet[DocumentAuthor] = ( + doc.rfcauthor_set.all() + if doc.rfcauthor_set.exists() + else doc.documentauthor_set.all() + ) + # RfcAuthorSerializer can deal with DocumentAuthor instances + return RfcAuthorSerializer( + instance=author_queryset, + many=True, + ).data + + @extend_schema_field(DocIdentifierSerializer(many=True)) + def get_identifiers(self, doc: Document): + identifiers = [] + if doc.doi: + identifiers.append( + DocIdentifier(type="doi", value=doc.doi) + ) + return DocIdentifierSerializer(instance=identifiers, many=True).data + + @extend_schema_field(RelatedDraftSerializer) + def get_draft(self, doc: Document): + if hasattr(doc, "drafts"): + # This is the expected case - drafts is added by a Prefetch in + # the augment_rfc_queryset() method. + try: + related_doc = doc.drafts[0] + except IndexError: + return None + else: + # Fallback in case augment_rfc_queryset() was not called + log.log( + f"Warning: {self.__class__}.get_draft() called without prefetched draft" + ) + related_doc = doc.came_from_draft() + return RelatedDraftSerializer(related_doc).data + + +class RfcSerializer(RfcMetadataSerializer): + """Serialize an RFC, including its metadata and text content if available""" + + text = serializers.CharField(allow_null=True) + + class Meta: + model = RfcMetadataSerializer.Meta.model + fields = RfcMetadataSerializer.Meta.fields + ["text"] + + +class SubseriesContentListSerializer(serializers.ListSerializer): + """ListSerializer that gets its object from item.target""" + + def to_representation(self, data): + """ + List of object instances -> List of dicts of primitive datatypes. + """ + # Dealing with nested relationships, data can be a Manager, + # so, first get a queryset from the Manager if needed + iterable = data.all() if isinstance(data, BaseManager) else data + # Serialize item.target instead of item itself + return [self.child.to_representation(item.target) for item in iterable] + + +class SubseriesContentSerializer(RfcMetadataSerializer): + """Serialize RFC contained in a subseries doc""" + + class Meta(RfcMetadataSerializer.Meta): + list_serializer_class = SubseriesContentListSerializer + + +class SubseriesDocSerializer(serializers.ModelSerializer): + """Serialize a subseries document (e.g., a BCP or STD)""" + + contents = SubseriesContentSerializer(many=True) + + class Meta: + model = Document + fields = [ + "name", + "type", + "contents", + ] diff --git a/ietf/doc/storage.py b/ietf/doc/storage.py index a234ef2d4f..375620ccaf 100644 --- a/ietf/doc/storage.py +++ b/ietf/doc/storage.py @@ -32,7 +32,7 @@ def __init__(self, file, name, mtime=None, content_type="", store=None, doc_name @classmethod def from_storedobject(cls, file, name, store): """Alternate constructor for objects that already exist in the StoredObject table""" - stored_object = StoredObject.objects.filter(store=store, name=name, deleted__isnull=True).first() + stored_object = StoredObject.objects.exclude_deleted().filter(store=store, name=name).first() if stored_object is None: raise FileNotFoundError(f"StoredObject for {store}:{name} does not exist or was deleted") file = cls(file, name, store, doc_name=stored_object.doc_name, doc_rev=stored_object.doc_rev) @@ -140,7 +140,11 @@ def _save_stored_object(self, name, content) -> StoredObject: ), ), ) - if not created: + if not created and ( + record.sha384 != content.custom_metadata["sha384"] + or record.len != int(content.custom_metadata["len"]) + or record.deleted is not None + ): record.sha384 = content.custom_metadata["sha384"] record.len = int(content.custom_metadata["len"]) record.modified = now @@ -160,7 +164,7 @@ def _delete_stored_object(self, name) -> Optional[StoredObject]: else: now = timezone.now() # Note that existing_record is a queryset that will have one matching object - existing_record.filter(deleted__isnull=True).update(deleted=now) + existing_record.exclude_deleted().update(deleted=now) return existing_record.first() def _save(self, name, content): diff --git a/ietf/doc/storage_utils.py b/ietf/doc/storage_utils.py index 510c98c4f5..81588c83ec 100644 --- a/ietf/doc/storage_utils.py +++ b/ietf/doc/storage_utils.py @@ -12,6 +12,14 @@ from ietf.utils.log import log +class StorageUtilsError(Exception): + pass + + +class AlreadyExistsError(StorageUtilsError): + pass + + def _get_storage(kind: str) -> Storage: if kind in settings.ARTIFACT_STORAGE_NAMES: return storages[kind] @@ -70,7 +78,7 @@ def store_file( # debug.show('f"Asked to store {name} in {kind}: is_new={is_new}, allow_overwrite={allow_overwrite}"') if not allow_overwrite and not is_new: debug.show('f"Failed to save {kind}:{name} - name already exists in store"') - raise RuntimeError(f"Failed to save {kind}:{name} - name already exists in store") + raise AlreadyExistsError(f"Failed to save {kind}:{name} - name already exists in store") new_name = _get_storage(kind).save( name, StoredObjectFile( @@ -85,7 +93,7 @@ def store_file( if new_name != name: complaint = f"Error encountered saving '{name}' - results stored in '{new_name}' instead." debug.show("complaint") - raise RuntimeError(complaint) + raise StorageUtilsError(complaint) except Exception as err: log(f"Blobstore Error: Failed to store file {kind}:{name}: {repr(err)}") if settings.SERVER_MODE == "development": diff --git a/ietf/doc/tasks.py b/ietf/doc/tasks.py index 4f7fe37782..90f4c80af5 100644 --- a/ietf/doc/tasks.py +++ b/ietf/doc/tasks.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2024-2025, All Rights Reserved +# Copyright The IETF Trust 2024-2026, All Rights Reserved # # Celery task definitions # @@ -29,10 +29,13 @@ from .utils import ( generate_idnits2_rfc_status, generate_idnits2_rfcs_obsoleted, + rebuild_reference_relations, update_or_create_draft_bibxml_file, ensure_draft_bibxml_path_exists, investigate_fragment, ) +from .utils_bofreq import fixup_bofreq_timestamps +from .utils_errata import signal_update_rfc_metadata @shared_task @@ -128,3 +131,32 @@ def investigate_fragment_task(name_fragment: str): "name_fragment": name_fragment, "results": investigate_fragment(name_fragment), } + +@shared_task +def rebuild_reference_relations_task(doc_names: list[str]): + log.log(f"Task: Rebuilding reference relations for {doc_names}") + for doc in Document.objects.filter(name__in=doc_names, type__in=["rfc", "draft"]): + filenames = dict() + base = ( + settings.RFC_PATH + if doc.type_id == "rfc" + else settings.INTERNET_ALL_DRAFTS_ARCHIVE_DIR + ) + stem = doc.name if doc.type_id == "rfc" else f"{doc.name}-{doc.rev}" + for ext in ["xml", "txt"]: + path = Path(base) / f"{stem}.{ext}" + if path.is_file(): + filenames[ext] = str(path) + if len(filenames) > 0: + rebuild_reference_relations(doc, filenames) + else: + log.log(f"Found no content for {stem}") + + +@shared_task +def fixup_bofreq_timestamps_task(): # pragma: nocover + fixup_bofreq_timestamps() + +@shared_task +def signal_update_rfc_metadata_task(rfc_number_list=()): + signal_update_rfc_metadata(rfc_number_list) diff --git a/ietf/doc/templatetags/ballot_icon.py b/ietf/doc/templatetags/ballot_icon.py index a94c145007..07a6c7f926 100644 --- a/ietf/doc/templatetags/ballot_icon.py +++ b/ietf/doc/templatetags/ballot_icon.py @@ -196,7 +196,7 @@ def state_age_colored(doc): .time ) except IndexError: - state_datetime = datetime.datetime(1990, 1, 1, tzinfo=datetime.timezone.utc) + state_datetime = datetime.datetime(1990, 1, 1, tzinfo=datetime.UTC) days = (timezone.now() - state_datetime).days # loosely based on the Publish Path page at the iesg wiki if iesg_state == "lc": diff --git a/ietf/doc/templatetags/ietf_filters.py b/ietf/doc/templatetags/ietf_filters.py index 5cabe1728d..ae5df641c2 100644 --- a/ietf/doc/templatetags/ietf_filters.py +++ b/ietf/doc/templatetags/ietf_filters.py @@ -1017,3 +1017,61 @@ def is_in_stream(doc): elif stream == "editorial": return True return False + + +@register.filter +def is_doc_ietf_adoptable(doc): + return doc.stream_id is None or all( + [ + doc.stream_id == "ietf", + doc.get_state_slug("draft-stream-ietf") + not in [ + "c-adopt", + "adopt-wg", + "info", + "wg-doc", + "parked", + "dead", + "wg-lc", + "waiting-for-implementation", + "chair-w", + "writeupw", + "sub-pub", + ], + doc.get_state_slug("draft") != "rfc", + doc.became_rfc() is None, + ] + ) + + +@register.filter +def can_issue_ietf_wg_lc(doc): + return all( + [ + doc.stream_id == "ietf", + doc.get_state_slug("draft-stream-ietf") + not in ["wg-cand", "c-adopt", "wg-lc"], + doc.get_state_slug("draft") != "rfc", + doc.became_rfc() is None, + ] + ) + + +@register.filter +def can_submit_to_iesg(doc): + return all( + [ + doc.stream_id == "ietf", + doc.get_state_slug("draft-iesg") == "idexists", + doc.get_state_slug("draft-stream-ietf") not in ["wg-cand", "c-adopt"], + ] + ) + + +@register.filter +def has_had_ietf_wg_lc(doc): + return ( + doc.stream_id == "ietf" + and doc.docevent_set.filter(statedocevent__state__slug="wg-lc").exists() + ) + diff --git a/ietf/doc/tests.py b/ietf/doc/tests.py index fa8c7fa4fc..f92c9648e6 100644 --- a/ietf/doc/tests.py +++ b/ietf/doc/tests.py @@ -39,11 +39,15 @@ from ietf.doc.models import ( Document, DocRelationshipName, RelatedDocument, State, DocEvent, BallotPositionDocEvent, LastCallDocEvent, WriteupDocEvent, NewRevisionDocEvent, BallotType, EditedAuthorsDocEvent, StateType) -from ietf.doc.factories import ( DocumentFactory, DocEventFactory, CharterFactory, - ConflictReviewFactory, WgDraftFactory, IndividualDraftFactory, WgRfcFactory, - IndividualRfcFactory, StateDocEventFactory, BallotPositionDocEventFactory, - BallotDocEventFactory, DocumentAuthorFactory, NewRevisionDocEventFactory, - StatusChangeFactory, DocExtResourceFactory, RgDraftFactory, BcpFactory) +from ietf.doc.factories import (DocumentFactory, DocEventFactory, CharterFactory, + ConflictReviewFactory, WgDraftFactory, + IndividualDraftFactory, WgRfcFactory, + IndividualRfcFactory, StateDocEventFactory, + BallotPositionDocEventFactory, + BallotDocEventFactory, DocumentAuthorFactory, + NewRevisionDocEventFactory, + StatusChangeFactory, DocExtResourceFactory, + RgDraftFactory, BcpFactory, RfcAuthorFactory) from ietf.doc.forms import NotifyForm from ietf.doc.fields import SearchableDocumentsField from ietf.doc.utils import ( @@ -449,17 +453,6 @@ def test_drafts_in_last_call(self): self.assertContains(r, draft.title) self.assertContains(r, escape(draft.action_holders.first().name)) - def test_in_iesg_process(self): - doc_in_process = IndividualDraftFactory() - doc_in_process.action_holders.set([PersonFactory()]) - doc_in_process.set_state(State.objects.get(type='draft-iesg', slug='lc')) - doc_not_in_process = IndividualDraftFactory() - r = self.client.get(urlreverse('ietf.doc.views_search.drafts_in_iesg_process')) - self.assertEqual(r.status_code, 200) - self.assertContains(r, doc_in_process.title) - self.assertContains(r, escape(doc_in_process.action_holders.first().name)) - self.assertNotContains(r, doc_not_in_process.title) - def test_indexes(self): draft = IndividualDraftFactory() rfc = WgRfcFactory() @@ -990,7 +983,7 @@ def test_edit_authors_permissions(self): # Relevant users not authorized to edit authors unauthorized_usernames = [ 'plain', - *[author.user.username for author in draft.authors()], + *[author.user.username for author in draft.author_persons()], draft.group.get_chair().person.user.username, 'ad' ] @@ -1005,7 +998,7 @@ def test_edit_authors_permissions(self): self.client.logout() # Try to add an author via POST - still only the secretary should be able to do this. - orig_authors = draft.authors() + orig_authors = draft.author_persons() post_data = self.make_edit_authors_post_data( basis='permission test', authors=draft.documentauthor_set.all(), @@ -1023,12 +1016,12 @@ def test_edit_authors_permissions(self): for username in unauthorized_usernames: login_testing_unauthorized(self, username, url, method='post', request_kwargs=dict(data=post_data)) draft = Document.objects.get(pk=draft.pk) - self.assertEqual(draft.authors(), orig_authors) # ensure draft author list was not modified + self.assertEqual(draft.author_persons(), orig_authors) # ensure draft author list was not modified login_testing_unauthorized(self, 'secretary', url, method='post', request_kwargs=dict(data=post_data)) r = self.client.post(url, post_data) self.assertEqual(r.status_code, 302) draft = Document.objects.get(pk=draft.pk) - self.assertEqual(draft.authors(), orig_authors + [new_auth_person]) + self.assertEqual(draft.author_persons(), orig_authors + [new_auth_person]) def make_edit_authors_post_data(self, basis, authors): """Helper to generate edit_authors POST data for a set of authors""" @@ -1376,8 +1369,8 @@ def test_edit_authors_edit_fields(self): basis=change_reason ) - old_address = draft.authors()[0].email() - new_email = EmailFactory(person=draft.authors()[0], address=f'changed-{old_address}') + old_address = draft.author_persons()[0].email() + new_email = EmailFactory(person=draft.author_persons()[0], address=f'changed-{old_address}') post_data['author-0-email'] = new_email.address post_data['author-1-affiliation'] = 'University of Nowhere' post_data['author-2-country'] = 'Chile' @@ -1410,17 +1403,17 @@ def test_edit_authors_edit_fields(self): country_event = change_events.filter(desc__icontains='changed country').first() self.assertIsNotNone(email_event) - self.assertIn(draft.authors()[0].name, email_event.desc) + self.assertIn(draft.author_persons()[0].name, email_event.desc) self.assertIn(before[0]['email'], email_event.desc) self.assertIn(after[0]['email'], email_event.desc) self.assertIsNotNone(affiliation_event) - self.assertIn(draft.authors()[1].name, affiliation_event.desc) + self.assertIn(draft.author_persons()[1].name, affiliation_event.desc) self.assertIn(before[1]['affiliation'], affiliation_event.desc) self.assertIn(after[1]['affiliation'], affiliation_event.desc) self.assertIsNotNone(country_event) - self.assertIn(draft.authors()[2].name, country_event.desc) + self.assertIn(draft.author_persons()[2].name, country_event.desc) self.assertIn(before[2]['country'], country_event.desc) self.assertIn(after[2]['country'], country_event.desc) @@ -1874,13 +1867,63 @@ def test_document_ballot_needed_positions(self): def test_document_json(self): doc = IndividualDraftFactory() - + author = DocumentAuthorFactory(document=doc) + r = self.client.get(urlreverse("ietf.doc.views_doc.document_json", kwargs=dict(name=doc.name))) self.assertEqual(r.status_code, 200) data = r.json() - self.assertEqual(doc.name, data['name']) - self.assertEqual(doc.pages,data['pages']) + self.assertEqual(data["name"], doc.name) + self.assertEqual(data["pages"], doc.pages) + self.assertEqual( + data["authors"], + [ + { + "name": author.person.name, + "email": author.email.address, + "affiliation": author.affiliation, + } + ] + ) + + def test_document_json_rfc(self): + doc = IndividualRfcFactory() + old_style_author = DocumentAuthorFactory(document=doc) + url = urlreverse("ietf.doc.views_doc.document_json", kwargs=dict(name=doc.name)) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + data = r.json() + self.assertEqual(data["name"], doc.name) + self.assertEqual(data["pages"], doc.pages) + self.assertEqual( + data["authors"], + [ + { + "name": old_style_author.person.name, + "email": old_style_author.email.address, + "affiliation": old_style_author.affiliation, + } + ] + ) + + new_style_author = RfcAuthorFactory(document=doc) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + data = r.json() + self.assertEqual(data["name"], doc.name) + self.assertEqual(data["pages"], doc.pages) + self.assertEqual( + data["authors"], + [ + { + "name": new_style_author.titlepage_name, + "email": new_style_author.email.address, + "affiliation": new_style_author.affiliation, + } + ] + ) + + def test_writeup(self): doc = IndividualDraftFactory(states = [('draft','active'),('draft-iesg','iesg-eva')],) diff --git a/ietf/doc/tests_ballot.py b/ietf/doc/tests_ballot.py index 810ee598f6..8420e411e2 100644 --- a/ietf/doc/tests_ballot.py +++ b/ietf/doc/tests_ballot.py @@ -25,7 +25,6 @@ from ietf.group.models import Group, Role from ietf.group.factories import GroupFactory, RoleFactory, ReviewTeamFactory from ietf.ipr.factories import HolderIprDisclosureFactory -from ietf.name.models import BallotPositionName from ietf.iesg.models import TelechatDate from ietf.person.models import Person from ietf.person.factories import PersonFactory, PersonalApiKeyFactory @@ -37,9 +36,18 @@ class EditPositionTests(TestCase): + + # N.B. This test needs to be rewritten to exercise all types of ballots (iesg, irsg, rsab) + # and test against the output of the mailtriggers instead of looking for hardcoded values + # in the To and CC results. See #7864 def test_edit_position(self): ad = Person.objects.get(user__username="ad") - draft = IndividualDraftFactory(ad=ad,stream_id='ietf') + draft = WgDraftFactory( + ad=ad, + stream_id="ietf", + notify="somebody@example.com", + group__acronym="mars", + ) ballot = create_ballot_if_not_open(None, draft, ad, 'approve') url = urlreverse('ietf.doc.views_ballot.edit_position', kwargs=dict(name=draft.name, ballot_id=ballot.pk)) @@ -55,11 +63,20 @@ def test_edit_position(self): self.assertEqual(len(q('form textarea[name=comment]')), 1) # vote + empty_outbox() events_before = draft.docevent_set.count() - - r = self.client.post(url, dict(position="discuss", - discuss=" This is a discussion test. \n ", - comment=" This is a test. \n ")) + + r = self.client.post( + url, + dict( + position="discuss", + discuss=" This is a discussion test. \n ", + comment=" This is a test. \n ", + additional_cc="test298347@example.com", + cc_choices=["doc_notify", "doc_group_chairs"], + send_mail=1, + ), + ) self.assertEqual(r.status_code, 302) pos = draft.latest_event(BallotPositionDocEvent, balloter=ad) @@ -70,6 +87,22 @@ def test_edit_position(self): self.assertTrue(pos.comment_time != None) self.assertTrue("New position" in pos.desc) self.assertEqual(draft.docevent_set.count(), events_before + 3) + self.assertEqual(len(outbox),1) + m = outbox[0] + self.assertTrue("COMMENT" in m['Subject']) + self.assertTrue("DISCUSS" in m['Subject']) + self.assertTrue(draft.name in m['Subject']) + self.assertTrue("This is a discussion test." in str(m)) + self.assertTrue("This is a test" in str(m)) + self.assertTrue("iesg@" in m['To']) + # cc_choice doc_group_chairs + self.assertTrue("mars-chairs@" in m['Cc']) + # cc_choice doc_notify + self.assertTrue("somebody@example.com" in m['Cc']) + # cc_choice doc_group_email_list was not selected + self.assertFalse(draft.group.list_email in m['Cc']) + # extra-cc + self.assertTrue("test298347@example.com" in m['Cc']) # recast vote events_before = draft.docevent_set.count() @@ -230,64 +263,6 @@ def test_cannot_edit_position_as_pre_ad(self): r = self.client.post(url, dict(position="discuss", discuss="Test discuss text")) self.assertEqual(r.status_code, 403) - # N.B. This test needs to be rewritten to exercise all types of ballots (iesg, irsg, rsab) - # and test against the output of the mailtriggers instead of looking for hardcoded values - # in the To and CC results. See #7864 - def test_send_ballot_comment(self): - ad = Person.objects.get(user__username="ad") - draft = WgDraftFactory(ad=ad,group__acronym='mars') - draft.notify = "somebody@example.com" - draft.save_with_history([DocEvent.objects.create(doc=draft, rev=draft.rev, type="changed_document", by=Person.objects.get(user__username="secretary"), desc="Test")]) - - ballot = create_ballot_if_not_open(None, draft, ad, 'approve') - - BallotPositionDocEvent.objects.create( - doc=draft, rev=draft.rev, type="changed_ballot_position", - by=ad, balloter=ad, ballot=ballot, pos=BallotPositionName.objects.get(slug="discuss"), - discuss="This draft seems to be lacking a clearer title?", - discuss_time=timezone.now(), - comment="Test!", - comment_time=timezone.now()) - - url = urlreverse('ietf.doc.views_ballot.send_ballot_comment', kwargs=dict(name=draft.name, - ballot_id=ballot.pk)) - login_testing_unauthorized(self, "ad", url) - - # normal get - r = self.client.get(url) - self.assertEqual(r.status_code, 200) - q = PyQuery(r.content) - self.assertTrue(len(q('form input[name="extra_cc"]')) > 0) - - # send - mailbox_before = len(outbox) - - r = self.client.post(url, dict(extra_cc="test298347@example.com", cc_choices=['doc_notify','doc_group_chairs'])) - self.assertEqual(r.status_code, 302) - - self.assertEqual(len(outbox), mailbox_before + 1) - m = outbox[-1] - self.assertTrue("COMMENT" in m['Subject']) - self.assertTrue("DISCUSS" in m['Subject']) - self.assertTrue(draft.name in m['Subject']) - self.assertTrue("clearer title" in str(m)) - self.assertTrue("Test!" in str(m)) - self.assertTrue("iesg@" in m['To']) - # cc_choice doc_group_chairs - self.assertTrue("mars-chairs@" in m['Cc']) - # cc_choice doc_notify - self.assertTrue("somebody@example.com" in m['Cc']) - # cc_choice doc_group_email_list was not selected - self.assertFalse(draft.group.list_email in m['Cc']) - # extra-cc - self.assertTrue("test298347@example.com" in m['Cc']) - - r = self.client.post(url, dict(cc="")) - self.assertEqual(r.status_code, 302) - self.assertEqual(len(outbox), mailbox_before + 2) - m = outbox[-1] - self.assertTrue("iesg@" in m['To']) - self.assertFalse(m['Cc'] and draft.group.list_email in m['Cc']) class BallotWriteupsTests(TestCase): diff --git a/ietf/doc/tests_bofreq.py b/ietf/doc/tests_bofreq.py index 6a7c9393ef..6b142149be 100644 --- a/ietf/doc/tests_bofreq.py +++ b/ietf/doc/tests_bofreq.py @@ -307,17 +307,20 @@ def test_submit(self): url = urlreverse('ietf.doc.views_bofreq.submit', kwargs=dict(name=doc.name)) rev = doc.rev + doc_time = doc.time r = self.client.post(url,{'bofreq_submission':'enter','bofreq_content':'# oiwefrase'}) self.assertEqual(r.status_code, 302) doc = reload_db_objects(doc) - self.assertEqual(rev, doc.rev) + self.assertEqual(doc.rev, rev) + self.assertEqual(doc.time, doc_time) nobody = PersonFactory() self.client.login(username=nobody.user.username, password=nobody.user.username+'+password') r = self.client.post(url,{'bofreq_submission':'enter','bofreq_content':'# oiwefrase'}) self.assertEqual(r.status_code, 403) doc = reload_db_objects(doc) - self.assertEqual(rev, doc.rev) + self.assertEqual(doc.rev, rev) + self.assertEqual(doc.time, doc_time) self.client.logout() editor = bofreq_editors(doc).first() @@ -339,12 +342,14 @@ def test_submit(self): r = self.client.post(url, postdict) self.assertEqual(r.status_code, 302) doc = reload_db_objects(doc) - self.assertEqual('%02d'%(int(rev)+1) ,doc.rev) - self.assertEqual(f'# {username}', doc.text()) - self.assertEqual(f'# {username}', retrieve_str('bofreq',doc.get_base_name())) - self.assertEqual(docevent_count+1, doc.docevent_set.count()) - self.assertEqual(1, len(outbox)) + self.assertEqual(doc.rev, '%02d'%(int(rev)+1)) + self.assertGreater(doc.time, doc_time) + self.assertEqual(doc.text(), f'# {username}') + self.assertEqual(retrieve_str('bofreq', doc.get_base_name()), f'# {username}') + self.assertEqual(doc.docevent_set.count(), docevent_count+1) + self.assertEqual(len(outbox), 1) rev = doc.rev + doc_time = doc.time finally: os.unlink(file.name) diff --git a/ietf/doc/tests_draft.py b/ietf/doc/tests_draft.py index ab7eaba768..21a873c5c0 100644 --- a/ietf/doc/tests_draft.py +++ b/ietf/doc/tests_draft.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- +import json import os import datetime import io @@ -11,7 +12,7 @@ from pathlib import Path from pyquery import PyQuery -from django.db.models import Q +from django.db.models import Max, Q from django.urls import reverse as urlreverse from django.conf import settings from django.utils import timezone @@ -20,13 +21,14 @@ import debug # pyflakes:ignore from ietf.doc.expire import expirable_drafts, get_expired_drafts, send_expire_notice_for_draft, expire_draft -from ietf.doc.factories import EditorialDraftFactory, IndividualDraftFactory, WgDraftFactory, RgDraftFactory, DocEventFactory +from ietf.doc.factories import EditorialDraftFactory, IndividualDraftFactory, StateDocEventFactory, WgDraftFactory, RgDraftFactory, DocEventFactory, WgRfcFactory from ietf.doc.models import ( Document, DocReminder, DocEvent, ConsensusDocEvent, LastCallDocEvent, RelatedDocument, State, TelechatDocEvent, WriteupDocEvent, DocRelationshipName, IanaExpertDocEvent ) from ietf.doc.storage_utils import exists_in_storage, store_str from ietf.doc.utils import get_tags_for_stream_id, create_ballot_if_not_open -from ietf.doc.views_draft import AdoptDraftForm +from ietf.doc.views_draft import AdoptDraftForm, IssueCallForAdoptionForm, IssueWorkingGroupLastCallForm +from ietf.ietfauth.utils import has_role from ietf.name.models import DocTagName, RoleName from ietf.group.factories import GroupFactory, RoleFactory from ietf.group.models import Group, Role @@ -85,7 +87,7 @@ def test_ad_approved(self): self.assertTrue("Approved: " in outbox[-1]['Subject']) self.assertTrue(draft.name in outbox[-1]['Subject']) self.assertTrue('iesg@' in outbox[-1]['To']) - + def test_change_state(self): ad = Person.objects.get(user__username="ad") draft = WgDraftFactory( @@ -138,7 +140,7 @@ def test_change_state(self): self.assertEqual(draft.get_state_slug("draft-iesg"), "review-e") self.assertTrue(not draft.tags.filter(slug="ad-f-up")) self.assertTrue(draft.tags.filter(slug="need-rev")) - self.assertCountEqual(draft.action_holders.all(), [ad] + draft.authors()) + self.assertCountEqual(draft.action_holders.all(), [ad] + draft.author_persons()) self.assertEqual(draft.docevent_set.count(), events_before + 3) self.assertTrue("Test comment" in draft.docevent_set.all()[0].desc) self.assertTrue("Changed action holders" in draft.docevent_set.all()[1].desc) @@ -177,7 +179,7 @@ def test_pull_from_rfc_queue(self): states=[('draft-iesg','rfcqueue')], ) DocEventFactory(type='started_iesg_process',by=ad,doc=draft,rev=draft.rev,desc="Started IESG Process") - draft.action_holders.add(*(draft.authors())) + draft.action_holders.add(*(draft.author_persons())) url = urlreverse('ietf.doc.views_draft.change_state', kwargs=dict(name=draft.name)) login_testing_unauthorized(self, "secretary", url) @@ -277,7 +279,7 @@ def test_request_last_call(self): states=[('draft-iesg','ad-eval')], ) DocEventFactory(type='started_iesg_process',by=ad,doc=draft,rev=draft.rev,desc="Started IESG Process") - draft.action_holders.add(*(draft.authors())) + draft.action_holders.add(*(draft.author_persons())) self.client.login(username="secretary", password="secretary+password") url = urlreverse('ietf.doc.views_draft.change_state', kwargs=dict(name=draft.name)) @@ -678,11 +680,11 @@ def test_in_draft_expire_freeze(self): datetime.datetime.combine( ietf_monday - datetime.timedelta(days=1), datetime.time(0, 0, 0), - tzinfo=datetime.timezone.utc, + tzinfo=datetime.UTC, ) )) self.assertFalse(in_draft_expire_freeze( - datetime.datetime.combine(ietf_monday, datetime.time(0, 0, 0), tzinfo=datetime.timezone.utc) + datetime.datetime.combine(ietf_monday, datetime.time(0, 0, 0), tzinfo=datetime.UTC) )) def test_warn_expirable_drafts(self): @@ -1367,7 +1369,7 @@ def _test_changing_ah(action_holders, reason): _test_changing_ah([doc.ad, doc.shepherd.person], 'this is a first test') _test_changing_ah([doc.ad], 'this is a second test') - _test_changing_ah(doc.authors(), 'authors can do it, too') + _test_changing_ah(doc.author_persons(), 'authors can do it, too') _test_changing_ah([], 'clear it back out') def test_doc_change_action_holders_as_doc_manager(self): @@ -1707,11 +1709,7 @@ def test_adopt_document(self): self.assertEqual(draft.group, chair_role.group) self.assertEqual(draft.stream_id, stream_state_type_slug[type_id][13:]) # trim off "draft-stream-" self.assertEqual(draft.docevent_set.count() - events_before, 5) - self.assertEqual(len(outbox), 2) - self.assertTrue("Call For Adoption" in outbox[0]["Subject"]) - self.assertTrue(f"{chair_role.group.acronym}-chairs@" in outbox[0]['To']) - self.assertTrue(f"{draft.name}@" in outbox[0]['To']) - self.assertTrue(f"{chair_role.group.acronym}@" in outbox[0]['To']) + self.assertEqual(len(outbox), 1) # contents of outbox[1] are tested elsewhere # adopt @@ -2002,6 +2000,40 @@ def test_set_state(self): self.assertTrue("mars-chairs@ietf.org" in outbox[0].as_string()) self.assertTrue("marsdelegate@ietf.org" in outbox[0].as_string()) + def test_set_stream_state_to_wglc(self): + def _form_presents_state_option(response, state): + q = PyQuery(response.content) + option = q(f"select#id_new_state option[value='{state.pk}']") + return len(option) != 0 + + doc = WgDraftFactory() + chair = RoleFactory(name_id="chair", group=doc.group).person + url = urlreverse( + "ietf.doc.views_draft.change_stream_state", + kwargs=dict(name=doc.name, state_type="draft-stream-ietf"), + ) + login_testing_unauthorized(self, chair.user.username, url) + r = self.client.get(url) + wglc_state = State.objects.get(type="draft-stream-ietf", slug="wg-lc") + doc.set_state(wglc_state) + StateDocEventFactory( + doc=doc, + state_type_id="draft-stream-ietf", + state=("draft-stream-ietf", "wg-lc"), + ) + self.assertEqual(doc.docevent_set.count(), 2) + r = self.client.get(url) + self.assertTrue(_form_presents_state_option(r, wglc_state)) + other_doc = WgDraftFactory() + self.client.logout() + url = urlreverse( + "ietf.doc.views_draft.change_stream_state", + kwargs=dict(name=other_doc.name, state_type="draft-stream-ietf"), + ) + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + self.assertTrue(_form_presents_state_option(r, wglc_state)) + def test_wg_call_for_adoption_issued(self): role = RoleFactory( name_id="chair", @@ -2028,12 +2060,7 @@ def test_wg_call_for_adoption_issued(self): ), ) self.assertEqual(r.status_code, 302) - self.assertEqual(len(outbox), 2) - self.assertIn("mars-wg@ietf.org", outbox[1]["To"]) - self.assertIn("Call for adoption", outbox[1]["Subject"]) - body = get_payload_text(outbox[1]) - self.assertIn("disclosure obligations", body) - self.assertIn("starts a 10-week", body) + self.assertEqual(len(outbox), 1) # Test not entering a duration on the form draft = IndividualDraftFactory() url = urlreverse( @@ -2050,12 +2077,7 @@ def test_wg_call_for_adoption_issued(self): ), ) self.assertEqual(r.status_code, 302) - self.assertEqual(len(outbox), 2) - self.assertIn("mars-wg@ietf.org", outbox[1]["To"]) - self.assertIn("Call for adoption", outbox[1]["Subject"]) - body = get_payload_text(outbox[1]) - self.assertIn("disclosure obligations", body) - self.assertIn("starts a 2-week", body) + self.assertEqual(len(outbox), 1) # Test the less usual workflow of issuing a call for adoption # of a document that's already in the ietf stream @@ -2085,12 +2107,7 @@ def test_wg_call_for_adoption_issued(self): ), ) self.assertEqual(r.status_code, 302) - self.assertEqual(len(outbox), 2) - self.assertIn("mars-wg@ietf.org", outbox[1]["To"]) - self.assertIn("Call for adoption", outbox[1]["Subject"]) - body = get_payload_text(outbox[1]) - self.assertIn("disclosure obligations", body) - self.assertIn("starts a 10-week", body) + self.assertEqual(len(outbox), 1) draft = WgDraftFactory(group=role.group) url = urlreverse( "ietf.doc.views_draft.change_stream_state", @@ -2116,85 +2133,210 @@ def test_wg_call_for_adoption_issued(self): ), ) self.assertEqual(r.status_code, 302) - self.assertEqual(len(outbox), 2) - self.assertIn("mars-wg@ietf.org", outbox[1]["To"]) - self.assertIn("Call for adoption", outbox[1]["Subject"]) - body = get_payload_text(outbox[1]) - self.assertIn("disclosure obligations", body) - self.assertIn("starts a 2-week", body) + self.assertEqual(len(outbox), 1) - def test_wg_last_call_issued(self): - role = RoleFactory( - name_id="chair", - group__acronym="mars", - group__list_email="mars-wg@ietf.org", - person__user__username="marschairman", - person__name="WG Cháir Man", + def test_issue_wg_lc_form(self): + end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=1) + post = dict( + end_date=end_date, + to="foo@example.net, bar@example.com", + # Intentionally not passing cc + subject=f"garbage {end_date.isoformat()}", + body=f"garbage {end_date.isoformat()}", ) - draft = WgDraftFactory(group=role.group) - url = urlreverse( - "ietf.doc.views_draft.change_stream_state", - kwargs=dict(name=draft.name, state_type="draft-stream-ietf"), + form = IssueWorkingGroupLastCallForm(post) + self.assertTrue(form.is_valid()) + post["end_date"] = date_today(DEADLINE_TZINFO) + form = IssueWorkingGroupLastCallForm(post) + self.assertFalse(form.is_valid()) + self.assertIn( + "End date must be later than today", + form.errors["end_date"], + "Form accepted a too-early date", ) - login_testing_unauthorized(self, "marschairman", url) - old_state = draft.get_state("draft-stream-%s" % draft.stream_id) - new_state = State.objects.get( - used=True, type="draft-stream-%s" % draft.stream_id, slug="wg-lc" + post["end_date"] = end_date + datetime.timedelta(days=2) + form = IssueWorkingGroupLastCallForm(post) + self.assertFalse(form.is_valid()) + self.assertIn( + f"Last call end date ({post['end_date'].isoformat()}) not found in subject", + form.errors["subject"], + "form allowed subject without end_date", ) - self.assertNotEqual(old_state, new_state) + self.assertIn( + f"Last call end date ({post['end_date'].isoformat()}) not found in body", + form.errors["body"], + "form allowed body without end_date", + ) + + def test_issue_wg_lc(self): + def _assert_rejected(testcase, doc, person): + url = urlreverse( + "ietf.doc.views_draft.issue_wg_lc", kwargs=dict(name=doc.name) + ) + login_testing_unauthorized(testcase, person.user.username, url) + r = testcase.client.get(url) + testcase.assertEqual(r.status_code, 404) + testcase.client.logout() + + already_rfc = WgDraftFactory(states=[("draft", "rfc")]) + rfc_chair = RoleFactory(name_id="chair", group=already_rfc.group).person + _assert_rejected(self, already_rfc, rfc_chair) + rg_doc = RgDraftFactory() + rg_chair = RoleFactory(name_id="chair", group=rg_doc.group).person + _assert_rejected(self, rg_doc, rg_chair) + inwglc_doc = WgDraftFactory(states=[("draft-stream-ietf", "wg-lc")]) + inwglc_chair = RoleFactory(name_id="chair", group=inwglc_doc.group).person + _assert_rejected(self, inwglc_doc, inwglc_chair) + doc = WgDraftFactory() + chair = RoleFactory(name_id="chair", group=doc.group).person + url = urlreverse("ietf.doc.views_draft.issue_wg_lc", kwargs=dict(name=doc.name)) + login_testing_unauthorized(self, chair.user.username, url) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + postdict = dict() + postdict["end_date"] = q("input#id_end_date").attr("value") + postdict["to"] = q("input#id_to").attr("value") + ", extrato@example.org" + cc = q("input#id_cc").attr("value") + if cc is not None: + postdict["cc"] = cc + ", extracc@example.org" + else: + postdict["cc"] = "extracc@example.org" + postdict["subject"] = q("input#id_subject").attr("value") + " Extra Subject Words" + postdict["body"] = q("textarea#id_body").text() + "FGgqbQ$UNeXs" empty_outbox() r = self.client.post( url, - dict( - new_state=new_state.pk, - comment="some comment", - weeks="10", - tags=[ - t.pk - for t in draft.tags.filter( - slug__in=get_tags_for_stream_id(draft.stream_id) - ) - ], - ), + postdict, ) self.assertEqual(r.status_code, 302) + self.assertEqual(doc.get_state_slug("draft-stream-ietf"), "wg-lc") self.assertEqual(len(outbox), 2) - self.assertIn("mars-wg@ietf.org", outbox[1]["To"]) + self.assertIn(f"{doc.group.acronym}@ietf.org", outbox[1]["To"]) + self.assertIn("extrato@example.org", outbox[1]["To"]) + self.assertIn("extracc@example.org", outbox[1]["Cc"]) + self.assertIn("Extra Subject Words", outbox[1]["Subject"]) self.assertIn("WG Last Call", outbox[1]["Subject"]) body = get_payload_text(outbox[1]) self.assertIn("disclosure obligations", body) - self.assertIn("starts a 10-week", body) - draft = WgDraftFactory(group=role.group) - url = urlreverse( - "ietf.doc.views_draft.change_stream_state", - kwargs=dict(name=draft.name, state_type="draft-stream-ietf"), + self.assertIn("FGgqbQ$UNeXs", body) + + def test_issue_wg_call_for_adoption_form(self): + end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=1) + post = dict( + end_date=end_date, + to="foo@example.net, bar@example.com", + # Intentionally not passing cc + subject=f"garbage {end_date.isoformat()}", + body=f"garbage {end_date.isoformat()}", ) - old_state = draft.get_state("draft-stream-%s" % draft.stream_id) - new_state = State.objects.get( - used=True, type="draft-stream-%s" % draft.stream_id, slug="wg-lc" + form = IssueCallForAdoptionForm(post) + self.assertTrue(form.is_valid()) + post["end_date"] = date_today(DEADLINE_TZINFO) + form = IssueCallForAdoptionForm(post) + self.assertFalse(form.is_valid()) + self.assertIn( + "End date must be later than today", + form.errors["end_date"], + "Form accepted a too-early date", ) - self.assertNotEqual(old_state, new_state) - empty_outbox() - r = self.client.post( - url, - dict( - new_state=new_state.pk, - comment="some comment", - tags=[ - t.pk - for t in draft.tags.filter( - slug__in=get_tags_for_stream_id(draft.stream_id) - ) - ], - ), + post["end_date"] = end_date + datetime.timedelta(days=2) + form = IssueCallForAdoptionForm(post) + self.assertFalse(form.is_valid()) + self.assertIn( + f"Call for adoption end date ({post['end_date'].isoformat()}) not found in subject", + form.errors["subject"], + "form allowed subject without end_date", + ) + self.assertIn( + f"Call for adoption end date ({post['end_date'].isoformat()}) not found in body", + form.errors["body"], + "form allowed body without end_date", + ) + + def test_issue_wg_call_for_adoption(self): + def _assert_rejected(testcase, doc, person, group=None): + target_acronym = group.acronym if group is not None else doc.group.acronym + url = urlreverse( + "ietf.doc.views_draft.issue_wg_call_for_adoption", + kwargs=dict(name=doc.name, acronym=target_acronym), + ) + login_testing_unauthorized(testcase, person.user.username, url) + r = testcase.client.get(url) + testcase.assertEqual(r.status_code, 403) + testcase.client.logout() + + def _verify_call_issued(testcase, doc, chair_role): + url = urlreverse( + "ietf.doc.views_draft.issue_wg_call_for_adoption", + kwargs=dict(name=doc.name, acronym=chair_role.group.acronym), + ) + login_testing_unauthorized(testcase, chair_role.person.user.username, url) + r = testcase.client.get(url) + testcase.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + postdict = dict() + postdict["end_date"] = q("input#id_end_date").attr("value") + postdict["to"] = q("input#id_to").attr("value") + ", extrato@example.com" + self.assertIn(chair_role.group.list_email, postdict["to"]) + cc = q("input#id_cc").attr("value") + if cc is not None: + postdict["cc"] = cc + ", extracc@example.com" + else: + postdict["cc"] = "extracc@example.com" + postdict["subject"] = q("input#id_subject").attr("value") + " Extra Subject Words" + postdict["body"] = q("textarea#id_body").text() + "FGgqbQ$UNeXs" + empty_outbox() + r = testcase.client.post( + url, + postdict, + ) + testcase.assertEqual(r.status_code, 302) + doc.refresh_from_db() + self.assertEqual(doc.group, chair_role.group) + self.assertEqual(doc.get_state_slug("draft-stream-ietf"), "c-adopt") + self.assertEqual(len(outbox), 2) + self.assertIn(f"{doc.group.acronym}@ietf.org", outbox[1]["To"]) + self.assertIn("extrato@example.com", outbox[1]["To"]) + self.assertIn("extracc@example.com", outbox[1]["Cc"]) + self.assertIn("Call for adoption", outbox[1]["Subject"]) + self.assertIn("Extra Subject Words", outbox[1]["Subject"]) + body = get_payload_text(outbox[1]) + self.assertIn("disclosure obligations", body) + self.assertIn("FGgqbQ$UNeXs", body) + self.client.logout() + return doc + + already_rfc = WgDraftFactory(states=[("draft", "rfc")]) + rfc = WgRfcFactory(group=already_rfc.group) + already_rfc.relateddocument_set.create(relationship_id="became_rfc",target=rfc) + rfc_chair = RoleFactory(name_id="chair", group=already_rfc.group).person + _assert_rejected(self, already_rfc, rfc_chair) + rg_doc = RgDraftFactory() + rg_chair = RoleFactory(name_id="chair", group=rg_doc.group).person + _assert_rejected(self, rg_doc, rg_chair) + inwglc_doc = WgDraftFactory(states=[("draft-stream-ietf", "wg-lc")]) + inwglc_chair = RoleFactory(name_id="chair", group=inwglc_doc.group).person + _assert_rejected(self, inwglc_doc, inwglc_chair) + ind_doc = IndividualDraftFactory() + _assert_rejected(self, ind_doc, rg_chair, rg_doc.group) + + # Successful call issued for doc already in WG + doc = WgDraftFactory(states=[("draft-stream-ietf","wg-cand")]) + chair_role = RoleFactory(name_id="chair",group=doc.group) + _ = _verify_call_issued(self, doc, chair_role) + + # Successful call issued for doc not yet in WG + doc = IndividualDraftFactory() + chair_role = RoleFactory(name_id="chair",group__type_id="wg") + doc = _verify_call_issued(self, doc, chair_role) + self.assertEqual(doc.group, chair_role.group) + self.assertEqual(doc.stream_id, "ietf") + self.assertEqual(doc.get_state_slug("draft-stream-ietf"), "c-adopt") + self.assertCountEqual( + doc.docevent_set.values_list("type", flat=True), + ["changed_state", "changed_group", "changed_stream", "new_revision"] ) - self.assertEqual(r.status_code, 302) - self.assertEqual(len(outbox), 2) - self.assertIn("mars-wg@ietf.org", outbox[1]["To"]) - self.assertIn("WG Last Call", outbox[1]["Subject"]) - body = get_payload_text(outbox[1]) - self.assertIn("disclosure obligations", body) - self.assertIn("starts a 2-week", body) def test_pubreq_validation(self): role = RoleFactory(name_id='chair',group__acronym='mars',group__list_email='mars-wg@ietf.org',person__user__username='marschairman',person__name='WG Cháir Man') @@ -2391,3 +2533,259 @@ def test_editorial_metadata(self): top_level_metadata_headings = q("tbody>tr>th:first-child").text() self.assertNotIn("IESG", top_level_metadata_headings) self.assertNotIn("IANA", top_level_metadata_headings) + +class IetfGroupActionHelperTests(TestCase): + def test_manage_adoption_routing(self): + draft = IndividualDraftFactory() + nobody = PersonFactory() + rgchair = RoleFactory(group__type_id="rg", name_id="chair").person + wgchair = RoleFactory(group__type_id="wg", name_id="chair").person + multichair = RoleFactory(group__type_id="rg", name_id="chair").person + RoleFactory(group__type_id="wg", person=multichair, name_id="chair") + ad = RoleFactory(group__type_id="area", name_id="ad").person + secretary = Role.objects.filter( + name_id="secr", group__acronym="secretariat" + ).first() + self.assertIsNotNone(secretary) + secretary = secretary.person + self.assertFalse( + has_role(rgchair.user, ["Secretariat", "Area Director", "WG Chair"]) + ) + url = urlreverse( + "ietf.doc.views_doc.document_main", kwargs={"name": draft.name} + ) + ask_about_ietf_link = urlreverse( + "ietf.doc.views_draft.ask_about_ietf_adoption_call", + kwargs={"name": draft.name}, + ) + non_ietf_adoption_link = urlreverse( + "ietf.doc.views_draft.adopt_draft", kwargs={"name": draft.name} + ) + for person in (None, nobody, rgchair, wgchair, multichair, ad, secretary): + if person is not None: + self.client.login( + username=person.user.username, + password=f"{person.user.username}+password", + ) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + q = PyQuery(r.content) + has_ask_about_ietf_link = len(q(f'a[href="{ask_about_ietf_link}"]')) != 0 + has_non_ietf_adoption_link = ( + len(q(f'a[href="{non_ietf_adoption_link}"]')) != 0 + ) + ask_about_r = self.client.get(ask_about_ietf_link) + ask_about_link_return_code = ask_about_r.status_code + if person == rgchair: + self.assertFalse(has_ask_about_ietf_link) + self.assertTrue(has_non_ietf_adoption_link) + self.assertEqual(ask_about_link_return_code, 403) + elif person in (ad, nobody, None): + self.assertFalse(has_ask_about_ietf_link) + self.assertFalse(has_non_ietf_adoption_link) + self.assertEqual( + ask_about_link_return_code, 302 if person is None else 403 + ) + else: + self.assertTrue(has_ask_about_ietf_link) + self.assertFalse(has_non_ietf_adoption_link) + self.assertEqual(ask_about_link_return_code, 200) + self.client.logout() + + def test_ask_about_ietf_adoption_call(self): + # Basic permission tests above + doc = IndividualDraftFactory() + self.assertEqual(doc.docevent_set.count(), 1) + chair_role = RoleFactory(group__type_id="wg", name_id="chair") + chair = chair_role.person + group = chair_role.group + othergroup = GroupFactory(type_id="wg") + url = urlreverse( + "ietf.doc.views_draft.ask_about_ietf_adoption_call", + kwargs={"name": doc.name}, + ) + login_testing_unauthorized(self, chair.user.username, url) + r = self.client.post(url, {"group": othergroup.pk}) + self.assertEqual(r.status_code, 200) + r = self.client.post(url, {"group": group.pk}) + self.assertEqual(r.status_code, 302) + + def test_offer_wg_action_helpers(self): + def _assert_view_presents_buttons(testcase, response, expected): + q = PyQuery(response.content) + for id, expect in expected: + button = q(f"#{id}") + testcase.assertEqual( + len(button) != 0, + expect + ) + + # View rejects access + came_from_draft = WgDraftFactory(states=[("draft","rfc")]) + rfc = WgRfcFactory(group=came_from_draft.group) + came_from_draft.relateddocument_set.create(relationship_id="became_rfc",target=rfc) + rfc_chair = RoleFactory(name_id="chair", group=rfc.group).person + url = urlreverse("ietf.doc.views_draft.offer_wg_action_helpers", kwargs=dict(name=came_from_draft.name)) + login_testing_unauthorized(self, rfc_chair.user.username, url) + r = self.client.get(url) + self.assertEqual(r.status_code, 404) + self.client.logout() + rg_draft = RgDraftFactory() + rg_chair = RoleFactory(group=rg_draft.group, name_id="chair").person + url = urlreverse("ietf.doc.views_draft.offer_wg_action_helpers", kwargs=dict(name=rg_draft.name)) + login_testing_unauthorized(self, rg_chair.user.username, url) + r = self.client.get(url) + self.assertEqual(r.status_code,404) + self.client.logout() + + # View offers access + draft = WgDraftFactory() + chair = RoleFactory(group=draft.group, name_id="chair").person + url = urlreverse("ietf.doc.views_draft.offer_wg_action_helpers", kwargs=dict(name=draft.name)) + login_testing_unauthorized(self, chair.user.username, url) + r = self.client.get(url) + self.assertEqual(r.status_code,200) + _assert_view_presents_buttons( + self, + r, + [ + ("id_wgadopt_button", False), + ("id_wglc_button", True), + ("id_pubreq_button", True), + ], + ) + draft.set_state(State.objects.get(type_id="draft-stream-ietf", slug="wg-cand")) + r = self.client.get(url) + self.assertEqual(r.status_code,200) + _assert_view_presents_buttons( + self, + r, + [ + ("id_wgadopt_button", True), + ("id_wglc_button", False), + ("id_pubreq_button", False), + ], + ) + draft.set_state(State.objects.get(type_id="draft-stream-ietf", slug="wg-lc")) + StateDocEventFactory( + doc=draft, + state_type_id="draft-stream-ietf", + state=("draft-stream-ietf", "wg-lc"), + ) + self.assertEqual(draft.docevent_set.count(), 2) + r = self.client.get(url) + self.assertEqual(r.status_code,200) + _assert_view_presents_buttons( + self, + r, + [ + ("id_wgadopt_button", False), + ("id_wglc_button", False), + ("id_pubreq_button", True), + ], + ) + draft.set_state(State.objects.get(type_id="draft-stream-ietf",slug="chair-w")) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + _assert_view_presents_buttons( + self, + r, + [ + ("id_wgadopt_button", False), + ("id_wglc_button", True), + ("id_pubreq_button", True), + ], + ) + self.assertContains(response=r,text="Issue Another Working Group Last Call", status_code=200) + other_draft = WgDraftFactory() + self.client.logout() + url = urlreverse("ietf.doc.views_draft.offer_wg_action_helpers", kwargs=dict(name=other_draft.name)) + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + _assert_view_presents_buttons( + self, + r, + [ + ("id_wgadopt_button", False), + ("id_wglc_button", True), + ("id_pubreq_button", True), + ], + ) + self.assertContains( + response=r, text="Issue Working Group Last Call", status_code=200 + ) + +class BallotEmailAjaxTests(TestCase): + def test_ajax_build_position_email(self): + def _post_json(self, url, json_to_post): + r = self.client.post( + url, json.dumps(json_to_post), content_type="application/json" + ) + self.assertEqual(r.status_code, 200) + return json.loads(r.content) + + doc = WgDraftFactory() + ad = RoleFactory( + name_id="ad", group=doc.group, person__name="Some Areadirector" + ).person + url = urlreverse("ietf.doc.views_ballot.ajax_build_position_email") + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + self.assertEqual(r.status_code, 405) + response = _post_json(self, url, {}) + self.assertFalse(response["success"]) + self.assertEqual(response["errors"], ["post_data not provided"]) + response = _post_json(self, url, {"dictis": "not empty"}) + self.assertFalse(response["success"]) + self.assertEqual(response["errors"], ["post_data not provided"]) + response = _post_json(self, url, {"post_data": {}}) + self.assertFalse(response["success"]) + self.assertEqual(len(response["errors"]), 7) + response = _post_json( + self, + url, + { + "post_data": { + "discuss": "aaaaaa", + "comment": "bbbbbb", + "position": "discuss", + "balloter": Person.objects.aggregate(maxpk=Max("pk") + 1)["maxpk"], + "docname": "this-draft-does-not-exist", + "cc_choices": ["doc_group_mail_list"], + "additional_cc": "foo@example.com", + } + }, + ) + self.assertFalse(response["success"]) + self.assertEqual( + response["errors"], + ["No person found matching balloter", "No document found matching docname"], + ) + response = _post_json( + self, + url, + { + "post_data": { + "discuss": "aaaaaa", + "comment": "bbbbbb", + "position": "discuss", + "balloter": ad.pk, + "docname": doc.name, + "cc_choices": ["doc_group_mail_list"], + "additional_cc": "foo@example.com", + } + }, + ) + self.assertTrue(response["success"]) + for snippet in [ + "aaaaaa", + "bbbbbb", + "DISCUSS", + ad.plain_name(), + doc.name, + doc.group.list_email, + "foo@example.com", + ]: + self.assertIn(snippet, response["text"]) + diff --git a/ietf/doc/tests_irsg_ballot.py b/ietf/doc/tests_irsg_ballot.py index aa62d8aaf9..d96cf9dbef 100644 --- a/ietf/doc/tests_irsg_ballot.py +++ b/ietf/doc/tests_irsg_ballot.py @@ -355,28 +355,35 @@ def test_issue_ballot(self): def test_take_and_email_position(self): draft = RgDraftFactory() ballot = IRSGBallotDocEventFactory(doc=draft) - url = urlreverse('ietf.doc.views_ballot.edit_position', kwargs=dict(name=draft.name, ballot_id=ballot.pk)) + self.balloter + url = ( + urlreverse( + "ietf.doc.views_ballot.edit_position", + kwargs=dict(name=draft.name, ballot_id=ballot.pk), + ) + + self.balloter + ) empty_outbox() login_testing_unauthorized(self, self.username, url) r = self.client.get(url) self.assertEqual(r.status_code, 200) - r = self.client.post(url, dict(position='yes', comment='oib239sb', send_mail='Save and send email')) + empty_outbox() + r = self.client.post( + url, + dict( + position="yes", + comment="oib239sb", + send_mail="Save and send email", + cc_choices=["doc_authors", "doc_group_chairs", "doc_group_mail_list"], + ), + ) self.assertEqual(r.status_code, 302) e = draft.latest_event(BallotPositionDocEvent) - self.assertEqual(e.pos.slug,'yes') - self.assertEqual(e.comment, 'oib239sb') - - url = urlreverse('ietf.doc.views_ballot.send_ballot_comment', kwargs=dict(name=draft.name, ballot_id=ballot.pk)) + self.balloter - - r = self.client.get(url) - self.assertEqual(r.status_code, 200) - - r = self.client.post(url, dict(cc_choices=['doc_authors','doc_group_chairs','doc_group_mail_list'], body="Stuff")) - self.assertEqual(r.status_code, 302) - self.assertEqual(len(outbox),1) - self.assertNotIn('discuss-criteria', get_payload_text(outbox[0])) + self.assertEqual(e.pos.slug, "yes") + self.assertEqual(e.comment, "oib239sb") + self.assertEqual(len(outbox), 1) + self.assertNotIn("discuss-criteria", get_payload_text(outbox[0])) def test_close_ballot(self): draft = RgDraftFactory() @@ -482,27 +489,31 @@ def test_cant_take_position_on_iesg_ballot(self): def test_take_and_email_position(self): draft = RgDraftFactory() ballot = IRSGBallotDocEventFactory(doc=draft) - url = urlreverse('ietf.doc.views_ballot.edit_position', kwargs=dict(name=draft.name, ballot_id=ballot.pk)) + url = urlreverse( + "ietf.doc.views_ballot.edit_position", + kwargs=dict(name=draft.name, ballot_id=ballot.pk), + ) empty_outbox() login_testing_unauthorized(self, self.username, url) r = self.client.get(url) self.assertEqual(r.status_code, 200) - r = self.client.post(url, dict(position='yes', comment='oib239sb', send_mail='Save and send email')) + r = self.client.post( + url, + dict( + position="yes", + comment="oib239sb", + send_mail="Save and send email", + cc_choices=["doc_authors", "doc_group_chairs", "doc_group_mail_list"], + ), + ) self.assertEqual(r.status_code, 302) e = draft.latest_event(BallotPositionDocEvent) - self.assertEqual(e.pos.slug,'yes') - self.assertEqual(e.comment, 'oib239sb') - - url = urlreverse('ietf.doc.views_ballot.send_ballot_comment', kwargs=dict(name=draft.name, ballot_id=ballot.pk)) - - r = self.client.get(url) - self.assertEqual(r.status_code, 200) - - r = self.client.post(url, dict(cc_choices=['doc_authors','doc_group_chairs','doc_group_mail_list'], body="Stuff")) + self.assertEqual(e.pos.slug, "yes") + self.assertEqual(e.comment, "oib239sb") self.assertEqual(r.status_code, 302) - self.assertEqual(len(outbox),1) + self.assertEqual(len(outbox), 1) class IESGMemberTests(TestCase): diff --git a/ietf/doc/tests_review.py b/ietf/doc/tests_review.py index 8c1fc99ffe..82d1b5c232 100644 --- a/ietf/doc/tests_review.py +++ b/ietf/doc/tests_review.py @@ -822,7 +822,7 @@ def test_complete_review_upload_content(self): r = self.client.get(url) self.assertEqual(r.status_code, 200) self.assertContains(r, assignment.review_request.team.list_email) - for author in assignment.review_request.doc.authors(): + for author in assignment.review_request.doc.author_persons(): self.assertContains(r, author.formatted_email()) # faulty post diff --git a/ietf/doc/tests_rsab_ballot.py b/ietf/doc/tests_rsab_ballot.py index 028f548232..9086106ba9 100644 --- a/ietf/doc/tests_rsab_ballot.py +++ b/ietf/doc/tests_rsab_ballot.py @@ -333,34 +333,19 @@ def test_take_and_email_position(self): r = self.client.get(url) self.assertEqual(r.status_code, 200) - r = self.client.post( - url, - dict(position="yes", comment="oib239sb", send_mail="Save and send email"), - ) - self.assertEqual(r.status_code, 302) - e = draft.latest_event(BallotPositionDocEvent) - self.assertEqual(e.pos.slug, "yes") - self.assertEqual(e.comment, "oib239sb") - - url = ( - urlreverse( - "ietf.doc.views_ballot.send_ballot_comment", - kwargs=dict(name=draft.name, ballot_id=ballot.pk), - ) - + self.balloter - ) - - r = self.client.get(url) - self.assertEqual(r.status_code, 200) - r = self.client.post( url, dict( + position="yes", + comment="oib239sb", + send_mail="Save and send email", cc_choices=["doc_authors", "doc_group_chairs", "doc_group_mail_list"], - body="Stuff", ), ) self.assertEqual(r.status_code, 302) + e = draft.latest_event(BallotPositionDocEvent) + self.assertEqual(e.pos.slug, "yes") + self.assertEqual(e.comment, "oib239sb") self.assertEqual(len(outbox), 1) self.assertNotIn("discuss-criteria", get_payload_text(outbox[0])) @@ -532,31 +517,19 @@ def test_take_and_email_position(self): r = self.client.get(url) self.assertEqual(r.status_code, 200) - r = self.client.post( - url, - dict(position="yes", comment="oib239sb", send_mail="Save and send email"), - ) - self.assertEqual(r.status_code, 302) - e = draft.latest_event(BallotPositionDocEvent) - self.assertEqual(e.pos.slug, "yes") - self.assertEqual(e.comment, "oib239sb") - - url = urlreverse( - "ietf.doc.views_ballot.send_ballot_comment", - kwargs=dict(name=draft.name, ballot_id=ballot.pk), - ) - - r = self.client.get(url) - self.assertEqual(r.status_code, 200) - r = self.client.post( url, dict( + position="yes", + comment="oib239sb", + send_mail="Save and send email", cc_choices=["doc_authors", "doc_group_chairs", "doc_group_mail_list"], - body="Stuff", ), ) self.assertEqual(r.status_code, 302) + e = draft.latest_event(BallotPositionDocEvent) + self.assertEqual(e.pos.slug, "yes") + self.assertEqual(e.comment, "oib239sb") self.assertEqual(len(outbox), 1) diff --git a/ietf/doc/tests_utils.py b/ietf/doc/tests_utils.py index f610fe3d76..a2784bc85e 100644 --- a/ietf/doc/tests_utils.py +++ b/ietf/doc/tests_utils.py @@ -13,13 +13,15 @@ from ietf.group.factories import GroupFactory, RoleFactory from ietf.name.models import DocTagName from ietf.person.factories import PersonFactory +from ietf.doc.factories import BallotPositionDocEventFactory from ietf.utils.test_utils import TestCase, name_of_file_containing, reload_db_objects from ietf.person.models import Person from ietf.doc.factories import DocumentFactory, WgRfcFactory, WgDraftFactory from ietf.doc.models import State, DocumentActionHolder, DocumentAuthor from ietf.doc.utils import (update_action_holders, add_state_change_event, update_documentauthors, fuzzy_find_documents, rebuild_reference_relations, build_file_urls, - ensure_draft_bibxml_path_exists, update_or_create_draft_bibxml_file) + ensure_draft_bibxml_path_exists, update_or_create_draft_bibxml_file, + last_ballot_doc_revision) from ietf.utils.draft import Draft, PlaintextDraft from ietf.utils.xmldraft import XMLDraft @@ -148,7 +150,7 @@ def test_update_action_holders_resets_age(self): doc = self.doc_in_iesg_state('pub-req') doc.action_holders.set([self.ad]) dah = doc.documentactionholder_set.get(person=self.ad) - dah.time_added = datetime.datetime(2020, 1, 1, tzinfo=datetime.timezone.utc) # arbitrary date in the past + dah.time_added = datetime.datetime(2020, 1, 1, tzinfo=datetime.UTC) # arbitrary date in the past dah.save() right_now = timezone.now() @@ -387,13 +389,13 @@ def test_requires_txt_or_xml(self): result = rebuild_reference_relations(self.doc, {}) self.assertCountEqual(result.keys(), ['errors']) self.assertEqual(len(result['errors']), 1) - self.assertIn('No Internet-Draft text available', result['errors'][0], + self.assertIn('No file available', result['errors'][0], 'Error should be reported if no Internet-Draft file is given') result = rebuild_reference_relations(self.doc, {'md': 'cant-do-this.md'}) self.assertCountEqual(result.keys(), ['errors']) self.assertEqual(len(result['errors']), 1) - self.assertIn('No Internet-Draft text available', result['errors'][0], + self.assertIn('No file available', result['errors'][0], 'Error should be reported if no XML or plaintext file is given') @patch.object(XMLDraft, 'get_refs') @@ -533,3 +535,27 @@ def test_update_draft_bibxml_file(self, mock): self.assertEqual(mock.call_count, 1) self.assertEqual(mock.call_args, call(doc, "26")) self.assertEqual(ref_path.read_text(), "This\nis\nmy\nbibxml") + + +class LastBallotDocRevisionTests(TestCase): + def test_last_ballot_doc_revision(self): + now = timezone.now() + ad = Person.objects.get(user__username="ad") + bpde_with_null_send_email = BallotPositionDocEventFactory( + time=now - datetime.timedelta(minutes=30), + send_email=None, + ) + ballot = bpde_with_null_send_email.ballot + BallotPositionDocEventFactory( + ballot=ballot, + balloter=ad, + pos_id='noobj', + comment='Commentary', + comment_time=timezone.now(), + send_email=None, + ) + doc = bpde_with_null_send_email.doc + rev = bpde_with_null_send_email.rev + nobody = PersonFactory() + self.assertIsNone(last_ballot_doc_revision(doc, nobody)) + self.assertEqual(rev, last_ballot_doc_revision(doc, ad)) diff --git a/ietf/doc/urls.py b/ietf/doc/urls.py index 60255af856..61e94b2231 100644 --- a/ietf/doc/urls.py +++ b/ietf/doc/urls.py @@ -53,13 +53,13 @@ url(r'^ad/?$', views_search.ad_workload), url(r'^ad/(?P[^/]+)/?$', views_search.docs_for_ad), url(r'^ad2/(?P[\w.-]+)/$', RedirectView.as_view(url='/doc/ad/%(name)s/', permanent=True)), - url(r'^for_iesg/?$', views_search.docs_for_iesg), + url(r'^for_iesg/?$', RedirectView.as_view(pattern_name='ietf.doc.views_search.docs_for_iesg', permanent=False)), url(r'^rfc-status-changes/?$', views_status_change.rfc_status_changes), url(r'^start-rfc-status-change/(?:%(name)s/)?$' % settings.URL_REGEXPS, views_status_change.start_rfc_status_change), url(r'^bof-requests/?$', views_bofreq.bof_requests), url(r'^bof-requests/new/$', views_bofreq.new_bof_request), url(r'^statement/new/$', views_statement.new_statement), - url(r'^iesg/?$', views_search.drafts_in_iesg_process), + url(r'^iesg/?$', views_search.docs_for_iesg), url(r'^email-aliases/?$', views_doc.email_aliases), url(r'^downref/?$', views_downref.downref_registry), url(r'^downref/add/?$', views_downref.downref_registry_add), @@ -75,7 +75,7 @@ # This block should really all be at the idealized docs.ietf.org service url(r'^html/(?Pbcp[0-9]+?)(\.txt|\.html)?/?$', RedirectView.as_view(url=settings.RFC_EDITOR_INFO_BASE_URL+"%(name)s", permanent=False)), url(r'^html/(?Pstd[0-9]+?)(\.txt|\.html)?/?$', RedirectView.as_view(url=settings.RFC_EDITOR_INFO_BASE_URL+"%(name)s", permanent=False)), - url(r'^html/%(name)s(?:-%(rev)s)?(\.txt|\.html)?/?$' % settings.URL_REGEXPS, views_doc.document_html), + url(r'^html/%(name)s(?:-(?P[0-9]{2}(-[0-9]{2})?))?(\.txt|\.html)?/?$' % settings.URL_REGEXPS, views_doc.document_html), url(r'^id/%(name)s(?:-%(rev)s)?(?:\.(?P(txt|html|xml)))?/?$' % settings.URL_REGEXPS, views_doc.document_raw_id), url(r'^pdf/%(name)s(?:-%(rev)s)?(?:\.(?P[a-z]+))?/?$' % settings.URL_REGEXPS, views_doc.document_pdfized), @@ -93,6 +93,8 @@ url(r'^ballots/irsg/$', views_ballot.irsg_ballot_status), url(r'^ballots/rsab/$', views_ballot.rsab_ballot_status), + url(r'^build-position-email/$', views_ballot.ajax_build_position_email), + url(r'^(?P(bcp|std|fyi))/?$', views_search.index_subseries), url(r'^%(name)s(?:/%(rev)s)?/$' % settings.URL_REGEXPS, views_doc.document_main), @@ -111,7 +113,6 @@ url(r'^%(name)s/ballot/rsab/$' % settings.URL_REGEXPS, views_doc.document_rsab_ballot), url(r'^%(name)s/ballot/(?P[0-9]+)/$' % settings.URL_REGEXPS, views_doc.document_ballot), url(r'^%(name)s/ballot/(?P[0-9]+)/position/$' % settings.URL_REGEXPS, views_ballot.edit_position), - url(r'^%(name)s/ballot/(?P[0-9]+)/emailposition/$' % settings.URL_REGEXPS, views_ballot.send_ballot_comment), url(r'^%(name)s/(?:%(rev)s/)?doc.json$' % settings.URL_REGEXPS, views_doc.document_json), url(r'^%(name)s/ballotpopup/(?P[0-9]+)/$' % settings.URL_REGEXPS, views_doc.ballot_popup), url(r'^(?P[A-Za-z0-9._+-]+)/reviewrequest/', include("ietf.doc.urls_review")), @@ -124,6 +125,7 @@ url(r'^%(name)s/edit/info/$' % settings.URL_REGEXPS, views_draft.edit_info), url(r'^%(name)s/edit/requestresurrect/$' % settings.URL_REGEXPS, views_draft.request_resurrect), url(r'^%(name)s/edit/submit-to-iesg/$' % settings.URL_REGEXPS, views_draft.to_iesg), + url(r'^%(name)s/edit/issue-wg-lc/$' % settings.URL_REGEXPS, views_draft.issue_wg_lc), url(r'^%(name)s/edit/resurrect/$' % settings.URL_REGEXPS, views_draft.resurrect), url(r'^%(name)s/edit/addcomment/$' % settings.URL_REGEXPS, views_doc.add_comment), @@ -142,9 +144,13 @@ url(r'^%(name)s/edit/shepherdemail/$' % settings.URL_REGEXPS, views_draft.change_shepherd_email), url(r'^%(name)s/edit/shepherdwriteup/$' % settings.URL_REGEXPS, views_draft.edit_shepherd_writeup), url(r'^%(name)s/edit/requestpublication/$' % settings.URL_REGEXPS, views_draft.request_publication), + url(r'^%(name)s/edit/ask-about-ietf-adoption/$' % settings.URL_REGEXPS, views_draft.ask_about_ietf_adoption_call), url(r'^%(name)s/edit/adopt/$' % settings.URL_REGEXPS, views_draft.adopt_draft), + url(r'^%(name)s/edit/issue-wg-call-for-adoption/%(acronym)s/$' % settings.URL_REGEXPS, views_draft.issue_wg_call_for_adoption), + url(r'^%(name)s/edit/release/$' % settings.URL_REGEXPS, views_draft.release_draft), url(r'^%(name)s/edit/state/(?Pdraft-stream-[a-z]+)/$' % settings.URL_REGEXPS, views_draft.change_stream_state), + url(r'^%(name)s/edit/wg-action-helpers/$' % settings.URL_REGEXPS, views_draft.offer_wg_action_helpers), url(r'^%(name)s/edit/state/statement/$' % settings.URL_REGEXPS, views_statement.change_statement_state), url(r'^%(name)s/edit/clearballot/(?P[\w-]+)/$' % settings.URL_REGEXPS, views_ballot.clear_ballot), diff --git a/ietf/doc/utils.py b/ietf/doc/utils.py index 115b28b09b..396b3fcfa4 100644 --- a/ietf/doc/utils.py +++ b/ietf/doc/utils.py @@ -13,7 +13,7 @@ from dataclasses import dataclass from hashlib import sha384 from pathlib import Path -from typing import Iterator, Optional, Union +from typing import Iterator, Optional, Union, Iterable from zoneinfo import ZoneInfo from django.conf import settings @@ -33,14 +33,22 @@ from ietf.community.models import CommunityList from ietf.community.utils import docs_tracked_by_community_list -from ietf.doc.models import Document, DocHistory, State, DocumentAuthor, DocHistoryAuthor +from ietf.doc.models import ( + DocHistory, + DocHistoryAuthor, + Document, + DocumentAuthor, + RfcAuthor, + State, EditedRfcAuthorsDocEvent, +) from ietf.doc.models import RelatedDocument, RelatedDocHistory, BallotType, DocReminder from ietf.doc.models import DocEvent, ConsensusDocEvent, BallotDocEvent, IRSGBallotDocEvent, NewRevisionDocEvent, StateDocEvent -from ietf.doc.models import TelechatDocEvent, DocumentActionHolder, EditedAuthorsDocEvent +from ietf.doc.models import TelechatDocEvent, DocumentActionHolder, EditedAuthorsDocEvent, BallotPositionDocEvent from ietf.name.models import DocReminderTypeName, DocRelationshipName from ietf.group.models import Role, Group, GroupFeatures from ietf.ietfauth.utils import has_role, is_authorized_in_doc_stream, is_individual_draft_author, is_bofreq_editor from ietf.person.models import Email, Person +from ietf.person.utils import get_active_balloters from ietf.review.models import ReviewWish from ietf.utils import draft, log from ietf.utils.mail import parseaddr, send_mail @@ -533,7 +541,7 @@ def update_action_holders(doc, prev_state=None, new_state=None, prev_tags=None, doc.action_holders.clear() if tags.removed("need-rev"): # Removed the 'need-rev' tag - drop authors from the action holders list - DocumentActionHolder.objects.filter(document=doc, person__in=doc.authors()).delete() + DocumentActionHolder.objects.filter(document=doc, person__in=doc.author_persons()).delete() elif tags.added("need-rev"): # Remove the AD if we're asking for a new revision DocumentActionHolder.objects.filter(document=doc, person=doc.ad).delete() @@ -548,7 +556,7 @@ def update_action_holders(doc, prev_state=None, new_state=None, prev_tags=None, doc.action_holders.add(doc.ad) # Authors get the action if a revision is needed if tags.added("need-rev"): - for auth in doc.authors(): + for auth in doc.author_persons(): doc.action_holders.add(auth) # Now create an event if we changed the set @@ -560,6 +568,40 @@ def update_action_holders(doc, prev_state=None, new_state=None, prev_tags=None, ) +def _change_field_and_describe( + author: DocumentAuthor | RfcAuthor, + field: str, + newval, + field_display_name: str | None = None, +): + # make the change + oldval = getattr(author, field) + setattr(author, field, newval) + + was_empty = oldval is None or len(str(oldval)) == 0 + now_empty = newval is None or len(str(newval)) == 0 + + # describe the change + if oldval == newval: + return None + else: + if field_display_name is None: + field_display_name = field + + if was_empty and not now_empty: + return 'set {field} to "{new}"'.format( + field=field_display_name, new=newval + ) + elif now_empty and not was_empty: + return 'cleared {field} (was "{old}")'.format( + field=field_display_name, old=oldval + ) + else: + return 'changed {field} from "{old}" to "{new}"'.format( + field=field_display_name, old=oldval, new=newval + ) + + def update_documentauthors(doc, new_docauthors, by=None, basis=None): """Update the list of authors for a document @@ -572,27 +614,6 @@ def update_documentauthors(doc, new_docauthors, by=None, basis=None): used. These objects will not be saved, their attributes will be used to create new DocumentAuthor instances. (The document and order fields will be ignored.) """ - def _change_field_and_describe(auth, field, newval): - # make the change - oldval = getattr(auth, field) - setattr(auth, field, newval) - - was_empty = oldval is None or len(str(oldval)) == 0 - now_empty = newval is None or len(str(newval)) == 0 - - # describe the change - if oldval == newval: - return None - else: - if was_empty and not now_empty: - return 'set {field} to "{new}"'.format(field=field, new=newval) - elif now_empty and not was_empty: - return 'cleared {field} (was "{old}")'.format(field=field, old=oldval) - else: - return 'changed {field} from "{old}" to "{new}"'.format( - field=field, old=oldval, new=newval - ) - persons = [] changes = [] # list of change descriptions @@ -636,6 +657,123 @@ def _change_field_and_describe(auth, field, newval): ) for change in changes ] + +def update_rfcauthors( + rfc: Document, new_rfcauthors: Iterable[RfcAuthor], by: Person | None = None +) -> Iterable[EditedRfcAuthorsDocEvent]: + def _find_matching_author( + author_to_match: RfcAuthor, existing_authors: Iterable[RfcAuthor] + ) -> RfcAuthor | None: + """Helper to find a matching existing author""" + if author_to_match.person_id is not None: + for candidate in existing_authors: + if candidate.person_id == author_to_match.person_id: + return candidate + return None # no match + # author does not have a person, match on titlepage name + for candidate in existing_authors: + if candidate.titlepage_name == author_to_match.titlepage_name: + return candidate + return None # no match + + def _rfcauthor_from_documentauthor(docauthor: DocumentAuthor) -> RfcAuthor: + """Helper to create an equivalent RfcAuthor from a DocumentAuthor""" + return RfcAuthor( + document_id=docauthor.document_id, + titlepage_name=docauthor.person.plain_name(), # closest thing we have + is_editor=False, + person_id=docauthor.person_id, + affiliation=docauthor.affiliation, + country=docauthor.country, + order=docauthor.order, + ) + + # Is this the first time this document is getting an RfcAuthor? If so, the + # updates will need to account for the model change. + converting_from_docauthors = not rfc.rfcauthor_set.exists() + + if converting_from_docauthors: + original_authors = [ + _rfcauthor_from_documentauthor(da) for da in rfc.documentauthor_set.all() + ] + else: + original_authors = list(rfc.rfcauthor_set.all()) + + authors_to_commit = [] + changes = [] + for order, new_author in enumerate(new_rfcauthors): + matching_author = _find_matching_author(new_author, original_authors) + if matching_author is not None: + # Update existing matching author using new_author data + authors_to_commit.append(matching_author) + original_authors.remove(matching_author) # avoid reuse + # Describe changes to this author + author_changes = [] + # Update fields other than order + for field in ["titlepage_name", "is_editor", "affiliation", "country"]: + author_changes.append( + _change_field_and_describe( + matching_author, + field, + getattr(new_author, field), + # List titlepage_name as "name" in logs + "name" if field == "titlepage_name" else field, + ) + ) + # Update order + author_changes.append( + _change_field_and_describe(matching_author, "order", order + 1) + ) + matching_author.save() + author_change_summary = ", ".join( + [ch for ch in author_changes if ch is not None] + ) + if len(author_change_summary) > 0: + changes.append( + 'Changed author "{name}": {summary}'.format( + name=matching_author.titlepage_name, + summary=author_change_summary, + ) + ) + else: + # No author matched, so update the new_author and use that + new_author.document = rfc + new_author.order = order + 1 + new_author.save() + if new_author.person_id is not None: + person_desc = f"Person {new_author.person_id}" + else: + person_desc = "no Person linked" + changes.append( + f'Added "{new_author.titlepage_name}" ({person_desc}) as author' + ) + # Any authors left in original_authors are no longer in the list, so remove them + for removed_author in original_authors: + # Skip actual removal of old authors if we are converting from the + # DocumentAuthor models - the original_authors were just stand-ins anyway. + if not converting_from_docauthors: + removed_author.delete() + if removed_author.person_id is not None: + person_desc = f"Person {removed_author.person_id}" + else: + person_desc = "no Person linked" + changes.append( + f'Removed "{removed_author.titlepage_name}" ({person_desc}) as author' + ) + # Create DocEvents, but leave it up to caller to save + if by is None: + by = Person.objects.get(name="(System)") + return [ + EditedRfcAuthorsDocEvent( + type="edited_authors", + by=by, + doc=rfc, + desc=change, + ) + for change in changes + ] + + def update_reminder(doc, reminder_type_slug, event, due_date): reminder_type = DocReminderTypeName.objects.get(slug=reminder_type_slug) @@ -687,6 +825,22 @@ def nice_consensus(consensus): } return mapping[consensus] +def last_ballot_doc_revision(doc, person): + """ Return the document revision for the most recent ballot position + by the provided user. """ + ballot = doc.active_ballot() + if ballot is None or person is None: + return None + balloters = get_active_balloters(ballot.ballot_type) + if person not in balloters: + return None + position_queryset = BallotPositionDocEvent.objects.filter(type="changed_ballot_position", balloter=person, ballot=ballot).order_by("-time") + if not position_queryset.exists(): + return None + ballot_time = position_queryset.first().time + doc_rev = NewRevisionDocEvent.objects.filter(doc=doc, time__lte=ballot_time).order_by('-time').first().rev + return doc_rev + def has_same_ballot(doc, date1, date2=None): """ Test if the most recent ballot created before the end of date1 is the same as the most recent ballot created before the @@ -799,50 +953,66 @@ def rebuild_reference_relations(doc, filenames): filenames should be a dict mapping file ext (i.e., type) to the full path of each file. """ - if doc.type.slug != 'draft': + if doc.type.slug not in ["draft", "rfc"]: return None + + log.log(f"Rebuilding reference relations for {doc.name}") # try XML first - if 'xml' in filenames: - refs = XMLDraft(filenames['xml']).get_refs() - elif 'txt' in filenames: - filename = filenames['txt'] + if "xml" in filenames: + refs = XMLDraft(filenames["xml"]).get_refs() + elif "txt" in filenames: + filename = filenames["txt"] try: refs = draft.PlaintextDraft.from_file(filename).get_refs() except IOError as e: - return { 'errors': ["%s :%s" % (e.strerror, filename)] } + return {"errors": [f"{e.strerror}: {filename}"]} else: - return {'errors': ['No Internet-Draft text available for rebuilding reference relations. Need XML or plaintext.']} + return { + "errors": [ + "No file available for rebuilding reference relations. Need XML or plaintext." + ] + } - doc.relateddocument_set.filter(relationship__slug__in=['refnorm','refinfo','refold','refunk']).delete() + doc.relateddocument_set.filter( + relationship__slug__in=["refnorm", "refinfo", "refold", "refunk"] + ).delete() warnings = [] errors = [] unfound = set() - for ( ref, refType ) in refs.items(): + for ref, refType in refs.items(): refdoc = Document.objects.filter(name=ref) if not refdoc and re.match(r"^draft-.*-\d{2}$", ref): refdoc = Document.objects.filter(name=ref[:-3]) count = refdoc.count() if count == 0: - unfound.add( "%s" % ref ) + unfound.add("%s" % ref) continue elif count > 1: - errors.append("Too many Document objects found for %s"%ref) + errors.append("Too many Document objects found for %s" % ref) else: # Don't add references to ourself if doc != refdoc[0]: - RelatedDocument.objects.get_or_create( source=doc, target=refdoc[ 0 ], relationship=DocRelationshipName.objects.get( slug='ref%s' % refType ) ) + RelatedDocument.objects.get_or_create( + source=doc, + target=refdoc[0], + relationship=DocRelationshipName.objects.get( + slug="ref%s" % refType + ), + ) if unfound: - warnings.append('There were %d references with no matching Document'%len(unfound)) + warnings.append( + "There were %d references with no matching Document" % len(unfound) + ) ret = {} if errors: - ret['errors']=errors + ret["errors"] = errors if warnings: - ret['warnings']=warnings + ret["warnings"] = warnings if unfound: - ret['unfound']=list(unfound) + ret["unfound"] = list(unfound) return ret diff --git a/ietf/doc/utils_bofreq.py b/ietf/doc/utils_bofreq.py index aec8f60ad6..d01b039b8e 100644 --- a/ietf/doc/utils_bofreq.py +++ b/ietf/doc/utils_bofreq.py @@ -1,12 +1,149 @@ -# Copyright The IETF Trust 2021 All Rights Reserved +# Copyright The IETF Trust 2021-2026 All Rights Reserved +import datetime +from pathlib import Path -from ietf.doc.models import BofreqEditorDocEvent, BofreqResponsibleDocEvent +from django.conf import settings + +from ietf.doc.models import ( + BofreqEditorDocEvent, + BofreqResponsibleDocEvent, + DocEvent, + DocHistory, + Document, +) from ietf.person.models import Person +from ietf.utils import log + def bofreq_editors(bofreq): e = bofreq.latest_event(BofreqEditorDocEvent) return e.editors.all() if e else Person.objects.none() + def bofreq_responsible(bofreq): e = bofreq.latest_event(BofreqResponsibleDocEvent) - return e.responsible.all() if e else Person.objects.none() \ No newline at end of file + return e.responsible.all() if e else Person.objects.none() + + +def fixup_bofreq_timestamps(): # pragma: nocover + """Fixes bofreq event / document timestamps + + Timestamp errors resulted from the bug fixed by + https://github.com/ietf-tools/datatracker/pull/10333 + + Does not fix up -00 revs because the timestamps on these were not affected by + the bug. Replacing their timestamps creates a confusing event history because the + filesystem timestamp is usually a fraction of a second later than other events + created upon the initial rev creation. This causes the "New revision available" + event to appear _after_ these events in the history. Better to leave them as is. + """ + FIX_DEPLOYMENT_TIME = "2026-02-03T01:16:00+00:00" # 12.58.0 -> production + + def _get_doc_time(doc_name: str, rev: str): + path = Path(settings.BOFREQ_PATH) / f"{doc_name}-{rev}.md" + return datetime.datetime.fromtimestamp(path.stat().st_mtime, datetime.UTC) + + # Find affected DocEvents and DocHistories + new_bofreq_events = ( + DocEvent.objects.filter( + doc__type="bofreq", type="new_revision", time__lt=FIX_DEPLOYMENT_TIME + ) + .exclude(rev="00") # bug did not affect rev 00 events + .order_by("doc__name", "rev") + ) + log.log( + f"fixup_bofreq_timestamps: found {new_bofreq_events.count()} " + f"new_revision events before {FIX_DEPLOYMENT_TIME}" + ) + document_fixups = {} + for e in new_bofreq_events: + name = e.doc.name + rev = e.rev + filesystem_time = _get_doc_time(name, rev) + assert e.time < filesystem_time, ( + f"Rev {rev} event timestamp for {name} unexpectedly later than the " + "filesystem timestamp!" + ) + try: + dochistory = DocHistory.objects.filter( + name=name, time__lt=filesystem_time + ).get(rev=rev) + except DocHistory.MultipleObjectsReturned as err: + raise RuntimeError( + f"Multiple DocHistories for {name} rev {rev} exist earlier than the " + "filesystem timestamp!" + ) from err + except DocHistory.DoesNotExist as err: + if rev == "00": + # Unreachable because we don't adjust -00 revs, but could be needed + # if we did, in theory. In practice it's still not reached, but + # keeping the case for completeness. + dochistory = None + else: + raise RuntimeError( + f"No DocHistory for {name} rev {rev} exists earlier than the " + f"filesystem timestamp!" + ) from err + + if name not in document_fixups: + document_fixups[name] = [] + document_fixups[name].append( + { + "event": e, + "dochistory": dochistory, + "filesystem_time": filesystem_time, + } + ) + + # Now do the actual fixup + system_person = Person.objects.get(name="(System)") + for doc_name, fixups in document_fixups.items(): + bofreq = Document.objects.get(type="bofreq", name=doc_name) + log_msg_parts = [] + adjusted_revs = [] + for fixup in fixups: + event_to_fix = fixup["event"] + dh_to_fix = fixup["dochistory"] + new_time = fixup["filesystem_time"] + adjusted_revs.append(event_to_fix.rev) + + # Fix up the event + event_to_fix.time = new_time + event_to_fix.save() + log_msg_parts.append(f"rev {event_to_fix.rev} DocEvent") + + # Fix up the DocHistory + if dh_to_fix is not None: + dh_to_fix.time = new_time + dh_to_fix.save() + log_msg_parts.append(f"rev {dh_to_fix.rev} DocHistory") + + if event_to_fix.rev == bofreq.rev and bofreq.time < new_time: + # Update the Document without calling save(). Only update if + # the time has not changed so we don't inadvertently overwrite + # a concurrent update. + Document.objects.filter(pk=bofreq.pk, time=bofreq.time).update( + time=new_time + ) + bofreq.refresh_from_db() + if bofreq.rev == event_to_fix.rev: + log_msg_parts.append(f"rev {bofreq.rev} Document") + else: + log.log( + "fixup_bofreq_timestamps: WARNING: bofreq Document rev " + f"changed for {bofreq.name}" + ) + log.log(f"fixup_bofreq_timestamps: {bofreq.name}: " + ", ".join(log_msg_parts)) + + # Fix up the Document, if necessary, and add a record of the adjustment + DocEvent.objects.create( + type="added_comment", + by=system_person, + doc=bofreq, + rev=bofreq.rev, + desc=( + "Corrected inaccurate document and new revision event timestamps for " + + ("version " if len(adjusted_revs) == 1 else "versions ") + + ", ".join(adjusted_revs) + ), + ) diff --git a/ietf/doc/utils_errata.py b/ietf/doc/utils_errata.py new file mode 100644 index 0000000000..539262151f --- /dev/null +++ b/ietf/doc/utils_errata.py @@ -0,0 +1,35 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +import requests + +from django.conf import settings + +from ietf.utils.log import log + + +def signal_update_rfc_metadata(rfc_number_list=()): + key = getattr(settings, "ERRATA_METADATA_NOTIFICATION_API_KEY", None) + if key is not None: + headers = {"X-Api-Key": settings.ERRATA_METADATA_NOTIFICATION_API_KEY} + post_dict = { + "rfc_number_list": list(rfc_number_list), + } + try: + response = requests.post( + settings.ERRATA_METADATA_NOTIFICATION_URL, + headers=headers, + json=post_dict, + timeout=settings.DEFAULT_REQUESTS_TIMEOUT, + ) + except requests.Timeout as e: + log( + f"POST request timed out for {settings.ERRATA_METADATA_NOTIFICATION_URL} ]: {e}" + ) + # raise RuntimeError(f'POST request timed out for {settings.ERRATA_METADATA_NOTIFICATION_URL}') from e + return + if response.status_code != 200: + log( + f"POST request failed for {settings.ERRATA_METADATA_NOTIFICATION_URL} ]: {response.status_code} {response.text}" + ) + else: + log("No API key configured for errata metadata notification, skipping") diff --git a/ietf/doc/utils_search.py b/ietf/doc/utils_search.py index cfc8a872f8..a5f461f9bb 100644 --- a/ietf/doc/utils_search.py +++ b/ietf/doc/utils_search.py @@ -108,7 +108,10 @@ def fill_in_document_table_attributes(docs, have_telechat_date=False): d.search_heading = "Withdrawn Internet-Draft" d.expirable = False else: - d.search_heading = "%s Internet-Draft" % d.get_state() + if d.type_id == "draft" and d.stream_id == 'ietf' and d.get_state_slug('draft-iesg') != 'idexists': # values can be: ad-eval idexists approved rfcqueue dead iesg-eva + d.search_heading = "%s with the IESG Internet-Draft" % d.get_state() + else: + d.search_heading = "%s Internet-Draft" % d.get_state() if state_slug == "active": d.expirable = d.pk in expirable_pks else: @@ -221,6 +224,10 @@ def num(i): if d.type_id == "draft": res.append(num(["Active", "Expired", "Replaced", "Withdrawn", "RFC"].index(d.search_heading.split()[0]))) + if "with the IESG" in d.search_heading: + res.append("1") + else: + res.append("0") else: res.append(d.type_id); res.append("-"); diff --git a/ietf/doc/views_ballot.py b/ietf/doc/views_ballot.py index 0ba340890d..03cf01a4a1 100644 --- a/ietf/doc/views_ballot.py +++ b/ietf/doc/views_ballot.py @@ -4,18 +4,18 @@ # Directors and Secretariat -import datetime, json +import datetime +import json from django import forms from django.conf import settings -from django.http import HttpResponse, HttpResponseRedirect, Http404, HttpResponseBadRequest +from django.http import HttpResponse, HttpResponseNotAllowed, HttpResponseRedirect, Http404, HttpResponseBadRequest from django.shortcuts import render, get_object_or_404, redirect from django.template.defaultfilters import striptags from django.template.loader import render_to_string from django.urls import reverse as urlreverse from django.views.decorators.csrf import csrf_exempt from django.utils.html import escape -from urllib.parse import urlencode as urllib_urlencode import debug # pyflakes:ignore @@ -34,14 +34,15 @@ from ietf.doc.templatetags.ietf_filters import can_ballot from ietf.iesg.models import TelechatDate from ietf.ietfauth.utils import has_role, role_required, is_authorized_in_doc_stream +from ietf.mailtrigger.models import Recipient from ietf.mailtrigger.utils import gather_address_lists from ietf.mailtrigger.forms import CcSelectForm from ietf.message.utils import infer_message from ietf.name.models import BallotPositionName, DocTypeName from ietf.person.models import Person -from ietf.utils.fields import ModelMultipleChoiceField +from ietf.utils.fields import ModelMultipleChoiceField, MultiEmailField from ietf.utils.http import validate_return_to_path -from ietf.utils.mail import send_mail_text, send_mail_preformatted +from ietf.utils.mail import decode_header_value, send_mail_text, send_mail_preformatted from ietf.utils.decorators import require_api_key from ietf.utils.response import permission_denied from ietf.utils.timezone import date_today, datetime_from_date, DEADLINE_TZINFO @@ -179,6 +180,9 @@ def save_position(form, doc, ballot, balloter, login=None, send_email=False): return pos +class AdditionalCCForm(forms.Form): + additional_cc = MultiEmailField(required=False) + @role_required("Area Director", "Secretariat", "IRSG Member", "RSAB Member") def edit_position(request, name, ballot_id): """Vote and edit discuss and comment on document""" @@ -199,50 +203,67 @@ def edit_position(request, name, ballot_id): raise Http404 balloter = get_object_or_404(Person, pk=balloter_id) + if doc.stream_id == 'irtf': + mailtrigger_slug='irsg_ballot_saved' + elif doc.stream_id == 'editorial': + mailtrigger_slug='rsab_ballot_saved' + else: + mailtrigger_slug='iesg_ballot_saved' + if request.method == 'POST': old_pos = None if not has_role(request.user, "Secretariat") and not can_ballot(request.user, doc): # prevent pre-ADs from taking a position permission_denied(request, "Must be an active member (not a pre-AD for example) of the balloting body to take a position") + if request.POST.get("Defer") and doc.stream.slug != "irtf": + return redirect('ietf.doc.views_ballot.defer_ballot', name=doc) + elif request.POST.get("Undefer") and doc.stream.slug != "irtf": + return redirect('ietf.doc.views_ballot.undefer_ballot', name=doc) + form = EditPositionForm(request.POST, ballot_type=ballot.ballot_type) - if form.is_valid(): + cc_select_form = CcSelectForm(data=request.POST,mailtrigger_slug=mailtrigger_slug,mailtrigger_context={'doc':doc}) + additional_cc_form = AdditionalCCForm(request.POST) + if form.is_valid() and cc_select_form.is_valid() and additional_cc_form.is_valid(): send_mail = True if request.POST.get("send_mail") else False - save_position(form, doc, ballot, balloter, login, send_mail) - + pos = save_position(form, doc, ballot, balloter, login, send_mail) if send_mail: - query = {} - if request.GET.get('balloter'): - query['balloter'] = request.GET.get('balloter') - if request.GET.get('ballot_edit_return_point'): - query['ballot_edit_return_point'] = request.GET.get('ballot_edit_return_point') - qstr = "" - if len(query) > 0: - qstr = "?" + urllib_urlencode(query, safe='/') - return HttpResponseRedirect(urlreverse('ietf.doc.views_ballot.send_ballot_comment', kwargs=dict(name=doc.name, ballot_id=ballot_id)) + qstr) - elif request.POST.get("Defer") and doc.stream.slug != "irtf": - return redirect('ietf.doc.views_ballot.defer_ballot', name=doc) - elif request.POST.get("Undefer") and doc.stream.slug != "irtf": - return redirect('ietf.doc.views_ballot.undefer_ballot', name=doc) - else: - return HttpResponseRedirect(return_to_url) + addrs, frm, subject, body = build_position_email(balloter, doc, pos) + if doc.stream_id == 'irtf': + mailtrigger_slug='irsg_ballot_saved' + elif doc.stream_id == 'editorial': + mailtrigger_slug='rsab_ballot_saved' + else: + mailtrigger_slug='iesg_ballot_saved' + cc = [] + cc.extend(cc_select_form.get_selected_addresses()) + extra_cc = additional_cc_form.cleaned_data["additional_cc"] + if extra_cc: + cc.extend(extra_cc) + cc_set = set(cc) + cc_set.discard("") + cc = sorted(list(cc_set)) + send_mail_text(request, addrs.to, frm, subject, body, cc=", ".join(cc)) + return redirect(return_to_url) else: initial = {} old_pos = doc.latest_event(BallotPositionDocEvent, type="changed_ballot_position", balloter=balloter, ballot=ballot) if old_pos: initial['position'] = old_pos.pos.slug initial['discuss'] = old_pos.discuss - initial['comment'] = old_pos.comment - + initial['comment'] = old_pos.comment form = EditPositionForm(initial=initial, ballot_type=ballot.ballot_type) + cc_select_form = CcSelectForm(mailtrigger_slug=mailtrigger_slug,mailtrigger_context={'doc':doc}) + additional_cc_form = AdditionalCCForm() blocking_positions = dict((p.pk, p.name) for p in form.fields["position"].queryset.all() if p.blocking) - ballot_deferred = doc.active_defer_event() return render(request, 'doc/ballot/edit_position.html', dict(doc=doc, form=form, + cc_select_form=cc_select_form, + additional_cc_form=additional_cc_form, balloter=balloter, return_to_url=return_to_url, old_pos=old_pos, @@ -301,21 +322,98 @@ def err(code, text): ) -def build_position_email(balloter, doc, pos): +@role_required("Area Director", "Secretariat") +@csrf_exempt +def ajax_build_position_email(request): + if request.method != "POST": + return HttpResponseNotAllowed(["POST"]) + errors = list() + try: + json_body = json.loads(request.body) + except json.decoder.JSONDecodeError: + errors.append("Post body is not valid json") + if len(errors) == 0: + post_data = json_body.get("post_data") + if post_data is None: + errors.append("post_data not provided") + else: + for key in [ + "discuss", + "comment", + "position", + "balloter", + "docname", + "cc_choices", + "additional_cc", + ]: + if key not in post_data: + errors.append(f"{key} not found in post_data") + if len(errors) == 0: + person = Person.objects.filter(pk=post_data.get("balloter")).first() + if person is None: + errors.append("No person found matching balloter") + doc = Document.objects.filter(name=post_data.get("docname")).first() + if doc is None: + errors.append("No document found matching docname") + if len(errors) > 0: + response = { + "success": False, + "errors": errors, + } + else: + wanted = dict() # consider named tuple instead + wanted["discuss"] = post_data.get("discuss") + wanted["comment"] = post_data.get("comment") + wanted["position_name"] = post_data.get("position") + wanted["balloter"] = person + wanted["doc"] = doc + addrs, frm, subject, body = build_position_email_from_dict(wanted) + + recipient_slugs = post_data.get("cc_choices") + # Consider refactoring gather_address_lists so this isn't duplicated from there + cc_addrs = set() + for r in Recipient.objects.filter(slug__in=recipient_slugs): + cc_addrs.update(r.gather(doc=doc)) + additional_cc = post_data.get("additional_cc") + for addr in additional_cc.split(","): + cc_addrs.add(addr.strip()) + cc_addrs.discard("") + cc_addrs = sorted(list(cc_addrs)) + + response_text = "\n".join( + [ + f"From: {decode_header_value(frm)}", + f"To: {', '.join([decode_header_value(addr) for addr in addrs.to])}", + f"Cc: {', '.join([decode_header_value(addr) for addr in cc_addrs])}", + f"Subject: {subject}", + "", + body, + ] + ) + + response = { + "success": True, + "text": response_text, + } + return HttpResponse(json.dumps(response), content_type="application/json") + +def build_position_email_from_dict(pos_dict): + doc = pos_dict["doc"] subj = [] d = "" blocking_name = "DISCUSS" - if pos.pos.blocking and pos.discuss: - d = pos.discuss - blocking_name = pos.pos.name.upper() + pos_name = BallotPositionName.objects.filter(slug=pos_dict["position_name"]).first() + if pos_name.blocking and pos_dict.get("discuss"): + d = pos_dict.get("discuss") + blocking_name = pos_name.name.upper() subj.append(blocking_name) c = "" - if pos.comment: - c = pos.comment + if pos_dict.get("comment"): + c = pos_dict.get("comment") subj.append("COMMENT") - + balloter = pos_dict.get("balloter") balloter_name_genitive = balloter.plain_name() + "'" if balloter.plain_name().endswith('s') else balloter.plain_name() + "'s" - subject = "%s %s on %s" % (balloter_name_genitive, pos.pos.name if pos.pos else "No Position", doc.name + "-" + doc.rev) + subject = "%s %s on %s" % (balloter_name_genitive, pos_name.name if pos_name else "No Position", doc.name + "-" + doc.rev) if subj: subject += ": (with %s)" % " and ".join(subj) @@ -324,7 +422,7 @@ def build_position_email(balloter, doc, pos): comment=c, balloter=balloter.plain_name(), doc=doc, - pos=pos.pos, + pos=pos_name, blocking_name=blocking_name, settings=settings)) frm = balloter.role_email("ad").formatted_email() @@ -338,79 +436,16 @@ def build_position_email(balloter, doc, pos): return addrs, frm, subject, body -@role_required('Area Director','Secretariat','IRSG Member', 'RSAB Member') -def send_ballot_comment(request, name, ballot_id): - """Email document ballot position discuss/comment for Area Director.""" - doc = get_object_or_404(Document, name=name) - ballot = get_object_or_404(BallotDocEvent, type="created_ballot", pk=ballot_id, doc=doc) - if not has_role(request.user, 'Secretariat'): - if any([ - doc.stream_id == 'ietf' and not has_role(request.user, 'Area Director'), - doc.stream_id == 'irtf' and not has_role(request.user, 'IRSG Member'), - doc.stream_id == 'editorial' and not has_role(request.user, 'RSAB Member'), - ]): - raise Http404 - - balloter = request.user.person - - try: - return_to_url = parse_ballot_edit_return_point(request.GET.get('ballot_edit_return_point'), doc.name, ballot_id) - except ValueError: - return HttpResponseBadRequest('ballot_edit_return_point is invalid') - - if 'HTTP_REFERER' in request.META: - back_url = request.META['HTTP_REFERER'] - else: - back_url = urlreverse("ietf.doc.views_doc.document_ballot", kwargs=dict(name=doc.name, ballot_id=ballot_id)) - - # if we're in the Secretariat, we can select a balloter (such as an AD) to act as stand-in for - if has_role(request.user, "Secretariat"): - balloter_id = request.GET.get('balloter') - if not balloter_id: - raise Http404 - balloter = get_object_or_404(Person, pk=balloter_id) - - pos = doc.latest_event(BallotPositionDocEvent, type="changed_ballot_position", balloter=balloter, ballot=ballot) - if not pos: - raise Http404 - - addrs, frm, subject, body = build_position_email(balloter, doc, pos) - - if doc.stream_id == 'irtf': - mailtrigger_slug='irsg_ballot_saved' - elif doc.stream_id == 'editorial': - mailtrigger_slug='rsab_ballot_saved' - else: - mailtrigger_slug='iesg_ballot_saved' - - if request.method == 'POST': - cc = [] - cc_select_form = CcSelectForm(data=request.POST,mailtrigger_slug=mailtrigger_slug,mailtrigger_context={'doc':doc}) - if cc_select_form.is_valid(): - cc.extend(cc_select_form.get_selected_addresses()) - extra_cc = [x.strip() for x in request.POST.get("extra_cc","").split(',') if x.strip()] - if extra_cc: - cc.extend(extra_cc) - - send_mail_text(request, addrs.to, frm, subject, body, cc=", ".join(cc)) - - return HttpResponseRedirect(return_to_url) - - else: +def build_position_email(balloter, doc, pos): - cc_select_form = CcSelectForm(mailtrigger_slug=mailtrigger_slug,mailtrigger_context={'doc':doc}) - - return render(request, 'doc/ballot/send_ballot_comment.html', - dict(doc=doc, - subject=subject, - body=body, - frm=frm, - to=addrs.as_strings().to, - balloter=balloter, - back_url=back_url, - cc_select_form = cc_select_form, - )) + pos_dict=dict() + pos_dict["doc"]=doc + pos_dict["position_name"]=pos.pos.slug + pos_dict["discuss"]=pos.discuss + pos_dict["comment"]=pos.comment + pos_dict["balloter"]=balloter + return build_position_email_from_dict(pos_dict) @role_required('Area Director','Secretariat') def clear_ballot(request, name, ballot_type_slug): diff --git a/ietf/doc/views_bofreq.py b/ietf/doc/views_bofreq.py index 71cbe30491..94e3960dfa 100644 --- a/ietf/doc/views_bofreq.py +++ b/ietf/doc/views_bofreq.py @@ -91,7 +91,6 @@ def submit(request, name): by=request.user.person, rev=bofreq.rev, desc='New revision available', - time=bofreq.time, ) bofreq.save_with_history([e]) bofreq_submission = form.cleaned_data['bofreq_submission'] diff --git a/ietf/doc/views_doc.py b/ietf/doc/views_doc.py index 4a20db3c89..c1f6352ac3 100644 --- a/ietf/doc/views_doc.py +++ b/ietf/doc/views_doc.py @@ -79,6 +79,7 @@ from ietf.doc.views_ballot import parse_ballot_edit_return_point from ietf.doc.forms import InvestigateForm, TelechatForm, NotifyForm, ActionHoldersForm, DocAuthorForm, DocAuthorChangeBasisForm from ietf.doc.mails import email_comment, email_remind_action_holders +from ietf.doc.utils import last_ballot_doc_revision from ietf.mailtrigger.utils import gather_relevant_expansions from ietf.meeting.models import Session, SessionPresentation from ietf.meeting.utils import group_sessions, get_upcoming_manageable_sessions, sort_sessions, add_event_info_to_session_qs @@ -91,7 +92,7 @@ from ietf.utils.response import permission_denied from ietf.utils.text import maybe_split from ietf.utils.timezone import date_today - +from ietf.utils.unicodenormalize import normalize_for_sorting def render_document_top(request, doc, tab, name): tabs = [] @@ -514,13 +515,17 @@ def document_main(request, name, rev=None, document_html=False): # remaining actions actions = [] - if can_adopt_draft(request.user, doc) and not doc.get_state_slug() in ["rfc"] and not snapshot: + if can_adopt_draft(request.user, doc) and doc.get_state_slug() not in ["rfc"] and not snapshot: + target = urlreverse("ietf.doc.views_draft.adopt_draft", kwargs=dict(name=doc.name)) if doc.group and doc.group.acronym != 'none': # individual submission # already adopted in one group button_text = "Switch adoption" else: button_text = "Manage adoption" - actions.append((button_text, urlreverse('ietf.doc.views_draft.adopt_draft', kwargs=dict(name=doc.name)))) + # can_adopt_draft currently returns False for Area Directors + if has_role(request.user, ["Secretariat", "WG Chair"]): + target = urlreverse("ietf.doc.views_draft.ask_about_ietf_adoption_call", kwargs=dict(name=doc.name)) + actions.append((button_text, target)) if can_unadopt_draft(request.user, doc) and not doc.get_state_slug() in ["rfc"] and not snapshot: if doc.get_state_slug('draft-iesg') == 'idexists': @@ -1225,6 +1230,10 @@ def document_history(request, name): request.user, ("Area Director", "Secretariat", "IRTF Chair") ) + # if the current user has balloted on this document, give them a revision hint + ballot_doc_rev = None + if request.user.is_authenticated: + ballot_doc_rev = last_ballot_doc_revision(doc, request.user.person) return render( request, @@ -1235,6 +1244,7 @@ def document_history(request, name): "diff_revisions": diff_revisions, "events": events, "can_add_comment": can_add_comment, + "ballot_doc_rev": ballot_doc_rev, }, ) @@ -1275,9 +1285,7 @@ def document_bibtex(request, name, rev=None): break elif doc.type_id == "rfc": - # This needs to be replaced with a lookup, as the mapping may change - # over time. - doi = f"10.17487/RFC{doc.rfc_number:04d}" + doi = doc.doi if doc.is_dochistory(): latest_event = doc.latest_event(type='new_revision', rev=rev) @@ -1500,7 +1508,7 @@ def document_ballot_content(request, doc, ballot_id, editable=True): position_groups = [] for n in BallotPositionName.objects.filter(slug__in=[p.pos_id for p in positions]).order_by('order'): g = (n, [p for p in positions if p.pos_id == n.slug]) - g[1].sort(key=lambda p: (p.is_old_pos, p.balloter.plain_name())) + g[1].sort(key=lambda p: (p.is_old_pos, normalize_for_sorting(p.balloter.plain_name()))) if n.blocking: position_groups.insert(0, g) else: @@ -1643,11 +1651,18 @@ def extract_name(s): data["state"] = extract_name(doc.get_state()) data["intended_std_level"] = extract_name(doc.intended_std_level) data["std_level"] = extract_name(doc.std_level) + author_qs = ( + doc.rfcauthor_set + if doc.type_id == "rfc" and doc.rfcauthor_set.exists() + else doc.documentauthor_set + ).select_related("person").prefetch_related("person__email_set").order_by("order") data["authors"] = [ - dict(name=author.person.name, - email=author.email.address if author.email else None, - affiliation=author.affiliation) - for author in doc.documentauthor_set.all().select_related("person", "email").order_by("order") + { + "name": author.titlepage_name if hasattr(author, "titlepage_name") else author.person.name, + "email": author.email.address if author.email else None, + "affiliation": author.affiliation, + } + for author in author_qs ] data["shepherd"] = doc.shepherd.formatted_email() if doc.shepherd else None data["ad"] = doc.ad.role_email("ad").formatted_email() if doc.ad else None @@ -1931,9 +1946,9 @@ def edit_action_holders(request, name): role_ids = dict() # maps role slug to list of Person IDs (assumed numeric in the JavaScript) extra_prefetch = [] # list of Person objects to prefetch for select2 field - if len(doc.authors()) > 0: + authors = doc.author_persons() + if len(authors) > 0: doc_role_labels.append(dict(slug='authors', label='Authors')) - authors = doc.authors() role_ids['authors'] = [p.pk for p in authors] extra_prefetch += authors diff --git a/ietf/doc/views_draft.py b/ietf/doc/views_draft.py index 16d04ee66a..c5faf1140b 100644 --- a/ietf/doc/views_draft.py +++ b/ietf/doc/views_draft.py @@ -28,12 +28,12 @@ IanaExpertDocEvent, IESG_SUBSTATE_TAGS) from ietf.doc.mails import ( email_pulled_from_rfc_queue, email_resurrect_requested, email_resurrection_completed, email_state_changed, email_stream_changed, - email_wg_call_for_adoption_issued, email_wg_last_call_issued, email_stream_state_changed, email_stream_tags_changed, extra_automation_headers, generate_publication_request, email_adopted, email_intended_status_changed, email_iesg_processing_document, email_ad_approved_doc, email_iana_expert_review_state_changed ) from ietf.doc.storage_utils import retrieve_bytes, store_bytes +from ietf.doc.templatetags.ietf_filters import is_doc_ietf_adoptable from ietf.doc.utils import ( add_state_change_event, can_adopt_draft, can_unadopt_draft, get_tags_for_stream_id, nice_consensus, update_action_holders, update_reminder, update_telechat, make_notify_changed_event, get_initial_notify, @@ -51,12 +51,12 @@ from ietf.name.models import IntendedStdLevelName, DocTagName, StreamName from ietf.person.fields import SearchableEmailField from ietf.person.models import Person, Email -from ietf.utils.mail import send_mail, send_mail_message, on_behalf_of +from ietf.utils.mail import send_mail, send_mail_message, on_behalf_of, send_mail_text from ietf.utils.textupload import get_cleaned_text_file_content from ietf.utils import log -from ietf.utils.fields import ModelMultipleChoiceField +from ietf.utils.fields import DatepickerDateField, ModelMultipleChoiceField, MultiEmailField from ietf.utils.response import permission_denied -from ietf.utils.timezone import datetime_today, DEADLINE_TZINFO +from ietf.utils.timezone import date_today, datetime_from_date, datetime_today, DEADLINE_TZINFO class ChangeStateForm(forms.Form): @@ -1564,7 +1564,7 @@ def adopt_draft(request, name): events.append(e) due_date = None - if form.cleaned_data["weeks"] != None: + if form.cleaned_data["weeks"] is not None: due_date = datetime_today(DEADLINE_TZINFO) + datetime.timedelta(weeks=form.cleaned_data["weeks"]) update_reminder(doc, "stream-s", e, due_date) @@ -1573,11 +1573,6 @@ def adopt_draft(request, name): # setting states that are _not_ the adopted state. email_adopted(request, doc, prev_state, new_state, by, comment) - # Currently only the IETF stream uses the c-adopt state - guard against other - # streams starting to use it asthe IPR rules for those streams will be different. - if doc.stream_id == "ietf" and new_state.slug == "c-adopt": - email_wg_call_for_adoption_issued(request, doc, cfa_duration_weeks=form.cleaned_data["weeks"]) - # comment if comment: e = DocEvent(type="added_comment", doc=doc, rev=doc.rev, by=by) @@ -1689,11 +1684,14 @@ def __init__(self, *args, **kwargs): f.queryset = f.queryset.exclude(pk__in=unused_states) f.label = state_type.label if self.stream.slug == 'ietf': + help_text_items = [] if self.can_set_sub_pub: - f.help_text = "Only select 'Submitted to IESG for Publication' to correct errors. Use the document's main page to request publication." + help_text_items.append("Only select 'Submitted to IESG for Publication' to correct errors. This is not how to submit a document to the IESG.") else: f.queryset = f.queryset.exclude(slug='sub-pub') - f.help_text = "You may not set the 'Submitted to IESG for Publication' using this form - Use the document's main page to request publication." + help_text_items.append("You may not set the 'Submitted to IESG for Publication' using this form - Use the button above or the document's main page to request publication.") + help_text_items.append("Only use this form in unusual circumstances when issuing call for adoption or working group last call.") + f.help_text = " ".join(help_text_items) f = self.fields['tags'] f.queryset = f.queryset.filter(slug__in=get_tags_for_stream_id(doc.stream_id)) @@ -1704,7 +1702,7 @@ def __init__(self, *args, **kwargs): def clean_new_state(self): new_state = self.cleaned_data.get('new_state') if new_state.slug=='sub-pub' and not self.can_set_sub_pub: - raise forms.ValidationError('You may not set the %s state using this form. Use the "Submit to IESG for publication" button on the document\'s main page instead. If that button does not appear, the document may already have IESG state. Ask your Area Director or the Secretariat for help.'%new_state.name) + raise forms.ValidationError('You may not set the %s state using this form. Use the "Submit to IESG for Publication" button on the document\'s main page instead. If that button does not appear, the document may already have IESG state. Ask your Area Director or the Secretariat for help.'%new_state.name) return new_state @@ -1730,6 +1728,19 @@ def next_states_for_stream_state(doc, state_type, current_state): return next_states +@login_required +def offer_wg_action_helpers(request, name): + doc = get_object_or_404(Document, type="draft", name=name) + if doc.stream is None or doc.stream_id != "ietf" or doc.became_rfc() is not None: + raise Http404 + + if not is_authorized_in_doc_stream(request.user, doc): + permission_denied(request, "You don't have permission to access this page.") + + return render(request, "doc/draft/wg_action_helpers.html", + {"doc": doc, + }) + @login_required def change_stream_state(request, name, state_type): doc = get_object_or_404(Document, type="draft", name=name) @@ -1744,10 +1755,17 @@ def change_stream_state(request, name, state_type): prev_state = doc.get_state(state_type.slug) next_states = next_states_for_stream_state(doc, state_type, prev_state) + # These tell the form to allow directly setting the state to fix up errors. can_set_sub_pub = has_role(request.user,('Secretariat','Area Director')) or (prev_state and prev_state.slug=='sub-pub') if request.method == 'POST': - form = ChangeStreamStateForm(request.POST, doc=doc, state_type=state_type,can_set_sub_pub=can_set_sub_pub,stream=doc.stream) + form = ChangeStreamStateForm( + request.POST, + doc=doc, + state_type=state_type, + can_set_sub_pub=can_set_sub_pub, + stream=doc.stream, + ) if form.is_valid(): by = request.user.person events = [] @@ -1768,14 +1786,7 @@ def change_stream_state(request, name, state_type): update_reminder(doc, "stream-s", e, due_date) email_stream_state_changed(request, doc, prev_state, new_state, by, comment) - - if doc.stream_id == "ietf": - if new_state.slug == "c-adopt": - email_wg_call_for_adoption_issued(request, doc, cfa_duration_weeks=form.cleaned_data["weeks"]) - if new_state.slug == "wg-lc": - email_wg_last_call_issued(request, doc, wglc_duration_weeks=form.cleaned_data["weeks"]) - # tags existing_tags = set(doc.tags.all()) new_tags = set(form.cleaned_data["tags"]) @@ -1811,8 +1822,15 @@ def change_stream_state(request, name, state_type): else: form.add_error(None, "No change in state or tags found, and no comment provided -- nothing to do.") else: - form = ChangeStreamStateForm(initial=dict(new_state=prev_state.pk if prev_state else None, tags= doc.tags.all()), - doc=doc, state_type=state_type, can_set_sub_pub = can_set_sub_pub,stream = doc.stream) + form = ChangeStreamStateForm( + initial=dict( + new_state=prev_state.pk if prev_state else None, tags=doc.tags.all() + ), + doc=doc, + state_type=state_type, + can_set_sub_pub=can_set_sub_pub, + stream=doc.stream, + ) milestones = doc.groupmilestone_set.all() @@ -1857,3 +1875,325 @@ def set_intended_status_level(request, doc, new_level, old_level, comment): msg = "\n".join(e.desc for e in events) email_intended_status_changed(request, doc, msg) + +class IssueWorkingGroupLastCallForm(forms.Form): + end_date = DatepickerDateField( + required=True, + date_format="yyyy-mm-dd", + picker_settings={ + "autoclose": "1", + }, + help_text="The date the Last Call closes. If you change this, review the subject and body carefully to ensure the change is captured correctly.", + ) + + to = MultiEmailField( + required=True, + help_text="Comma separated list of address to use in the To: header", + ) + cc = MultiEmailField( + required=False, help_text="Comma separated list of addresses to copy" + ) + subject = forms.CharField( + required=True, + help_text="Subject for Last Call message. If you change the date here, be sure to make a matching change in the body.", + ) + body = forms.CharField( + widget=forms.Textarea, required=True, help_text="Body for Last Call message" + ) + + def clean_end_date(self): + end_date = self.cleaned_data["end_date"] + if end_date <= date_today(DEADLINE_TZINFO): + raise forms.ValidationError("End date must be later than today") + return end_date + + def clean(self): + cleaned_data = super().clean() + end_date = cleaned_data.get("end_date") + if end_date is not None: + body = cleaned_data.get("body") + subject = cleaned_data.get("subject") + if end_date.isoformat() not in body: + self.add_error( + "body", + forms.ValidationError( + f"Last call end date ({end_date.isoformat()}) not found in body" + ), + ) + if end_date.isoformat() not in subject: + self.add_error( + "subject", + forms.ValidationError( + f"Last call end date ({end_date.isoformat()}) not found in subject" + ), + ) + return cleaned_data + + +@login_required +def issue_wg_lc(request, name): + doc = get_object_or_404(Document, name=name) + + if doc.stream_id != "ietf": + raise Http404 + if doc.type_id != "draft" or doc.group.type_id != "wg": + raise Http404 + if doc.get_state_slug("draft-stream-ietf") == "wg-lc": + raise Http404 + if doc.get_state_slug("draft") == "rfc": + raise Http404 + + if not is_authorized_in_doc_stream(request.user, doc): + permission_denied(request, "You don't have permission to access this page.") + + if request.method == "POST": + form = IssueWorkingGroupLastCallForm(request.POST) + if form.is_valid(): + # Intentionally not changing tags or adding a comment + # those things can be done with other workflows + by = request.user.person + prev_state = doc.get_state("draft-stream-ietf") + events = [] + wglc_state = State.objects.get(type="draft-stream-ietf", slug="wg-lc") + doc.set_state(wglc_state) + e = add_state_change_event(doc, by, prev_state, wglc_state) + events.append(e) + end_date = form.cleaned_data["end_date"] + update_reminder( + doc, "stream-s", e, datetime_from_date(end_date, DEADLINE_TZINFO) + ) + doc.save_with_history(events) + email_stream_state_changed(request, doc, prev_state, wglc_state, by) + send_mail_text( + request, + to = form.cleaned_data["to"], + frm = request.user.person.formatted_email(), + subject = form.cleaned_data["subject"], + txt = form.cleaned_data["body"], + cc = form.cleaned_data["cc"], + ) + return redirect("ietf.doc.views_doc.document_main", name=doc.name) + else: + end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=14) + subject = f"WG Last Call: {doc.name}-{doc.rev} (Ends {end_date})" + body = render_to_string( + "doc/mail/wg_last_call_issued.txt", + dict( + doc=doc, + end_date=end_date, + wg_list=doc.group.list_email, + settings=settings, + ), + ) + (to, cc) = gather_address_lists("doc_wg_last_call_issued", doc=doc) + + form = IssueWorkingGroupLastCallForm( + initial=dict( + end_date=end_date, + to=", ".join(to), + cc=", ".join(cc), + subject=subject, + body=body, + ) + ) + + return render( + request, + "doc/draft/issue_working_group_last_call.html", + dict( + doc=doc, + form=form, + ), + ) + +class IssueCallForAdoptionForm(forms.Form): + end_date = DatepickerDateField( + required=True, + date_format="yyyy-mm-dd", + picker_settings={ + "autoclose": "1", + }, + help_text="The date the Call for Adoption closes. If you change this, review the subject and body carefully to ensure the change is captured correctly.", + ) + + to = MultiEmailField( + required=True, + help_text="Comma separated list of address to use in the To: header", + ) + cc = MultiEmailField( + required=False, help_text="Comma separated list of addresses to copy" + ) + subject = forms.CharField( + required=True, + help_text="Subject for Call for Adoption message. If you change the date here, be sure to make a matching change in the body.", + ) + body = forms.CharField( + widget=forms.Textarea, required=True, help_text="Body for Call for Adoption message" + ) + + def clean_end_date(self): + end_date = self.cleaned_data["end_date"] + if end_date <= date_today(DEADLINE_TZINFO): + raise forms.ValidationError("End date must be later than today") + return end_date + + def clean(self): + cleaned_data = super().clean() + end_date = cleaned_data.get("end_date") + if end_date is not None: + body = cleaned_data.get("body") + subject = cleaned_data.get("subject") + if end_date.isoformat() not in body: + self.add_error( + "body", + forms.ValidationError( + f"Call for adoption end date ({end_date.isoformat()}) not found in body" + ), + ) + if end_date.isoformat() not in subject: + self.add_error( + "subject", + forms.ValidationError( + f"Call for adoption end date ({end_date.isoformat()}) not found in subject" + ), + ) + return cleaned_data + +@login_required +def issue_wg_call_for_adoption(request, name, acronym): + doc = get_object_or_404(Document, name=name) + group = Group.objects.filter(acronym=acronym, type_id="wg").first() + reject = False + if group is None or doc.type_id != "draft" or not is_doc_ietf_adoptable(doc): + reject = True + if doc.stream is None: + if not can_adopt_draft(request.user, doc): + reject = True + elif doc.stream_id != "ietf": + reject = True + else: # doc.stream_id == "ietf" + if not is_authorized_in_doc_stream(request.user, doc): + reject = True + if reject: + raise permission_denied(request, f"You can't issue a {acronym} wg call for adoption for this document.") + + if request.method == "POST": + form = IssueCallForAdoptionForm(request.POST) + if form.is_valid(): + # Intentionally not changing tags or adding a comment + # those things can be done with other workflows + by = request.user.person + + events = [] + if doc.stream_id != "ietf": + stream = StreamName.objects.get(slug="ietf") + doc.stream = stream + e = DocEvent(type="changed_stream", doc=doc, rev=doc.rev, by=by) + e.desc = f"Changed stream to {stream.name}" # Propogates embedding html in DocEvent.desc for consistency + e.save() + events.append(e) + if doc.group != group: + doc.group = group + e = DocEvent(type="changed_group", doc=doc, rev=doc.rev, by=by) + e.desc = f"Changed group to {group.name} ({group.acronym.upper()})" # Even if it makes the cats cry + e.save() + events.append(e) + prev_state = doc.get_state("draft-stream-ietf") + c_adopt_state = State.objects.get(type="draft-stream-ietf", slug="c-adopt") + doc.set_state(c_adopt_state) + e = add_state_change_event(doc, by, prev_state, c_adopt_state) + events.append(e) + end_date = form.cleaned_data["end_date"] + update_reminder( + doc, "stream-s", e, datetime_from_date(end_date, DEADLINE_TZINFO) + ) + doc.save_with_history(events) + email_stream_state_changed(request, doc, prev_state, c_adopt_state, by) + send_mail_text( + request, + to = form.cleaned_data["to"], + frm = request.user.person.formatted_email(), + subject = form.cleaned_data["subject"], + txt = form.cleaned_data["body"], + cc = form.cleaned_data["cc"], + ) + return redirect("ietf.doc.views_doc.document_main", name=doc.name) + else: + end_date = date_today(DEADLINE_TZINFO) + datetime.timedelta(days=14) + subject = f"Call for adoption: {doc.name}-{doc.rev} (Ends {end_date})" + body = render_to_string( + "doc/mail/wg_call_for_adoption_issued.txt", + dict( + doc=doc, + group=group, + end_date=end_date, + wg_list=doc.group.list_email, + settings=settings, + ), + ) + (to, cc) = gather_address_lists("doc_wg_call_for_adoption_issued", doc=doc) + if doc.group.acronym == "none": + to.insert(0, f"{group.acronym}-chairs@ietf.org") + to.insert(0, group.list_email) + form = IssueCallForAdoptionForm( + initial=dict( + end_date=end_date, + to=", ".join(to), + cc=", ".join(cc), + subject=subject, + body=body, + ) + ) + + return render( + request, + "doc/draft/issue_working_group_call_for_adoption.html", + dict( + doc=doc, + form=form, + ), + ) + +class GroupModelChoiceField(forms.ModelChoiceField): + def label_from_instance(self, obj): + return f"{obj.acronym} - {obj.name}" + + +class WgForm(forms.Form): + group = GroupModelChoiceField( + queryset=Group.objects.filter(type_id="wg", state="active") + .order_by("acronym") + .distinct(), + required=True, + empty_label="Select IETF Working Group", + ) + + def __init__(self, *args, **kwargs): + user = kwargs.pop("user") + super(WgForm, self).__init__(*args, **kwargs) + if not has_role(user, ["Secretariat", "Area Director"]): + self.fields["group"].queryset = self.fields["group"].queryset.filter( + role__name_id="chair", role__person=user.person + ) + + +@role_required("Secretariat", "WG Chair") +def ask_about_ietf_adoption_call(request, name): + doc = get_object_or_404(Document, name=name) + if doc.stream is not None or doc.group.acronym != "none": + raise Http404 + if request.method == "POST": + form = WgForm(request.POST, user=request.user) + if form.is_valid(): + group = form.cleaned_data["group"] + return redirect(issue_wg_call_for_adoption, name=doc.name, acronym=group.acronym) + else: + form = WgForm(initial={"group": None}, user=request.user) + return render( + request, + "doc/draft/ask_about_ietf_adoption.html", + dict( + doc=doc, + form=form, + ), + ) diff --git a/ietf/doc/views_material.py b/ietf/doc/views_material.py index 6f8b8a8f12..eefac0ca61 100644 --- a/ietf/doc/views_material.py +++ b/ietf/doc/views_material.py @@ -22,6 +22,7 @@ from ietf.doc.utils import add_state_change_event, check_common_doc_name_rules from ietf.group.models import Group from ietf.group.utils import can_manage_materials +from ietf.meeting.utils import resolve_uploaded_material from ietf.utils import log from ietf.utils.decorators import ignore_view_kwargs from ietf.utils.meetecho import MeetechoAPIError, SlidesManager @@ -179,6 +180,9 @@ def edit_material(request, name=None, acronym=None, action=None, doc_type=None): "There was an error creating a hardlink at %s pointing to %s: %s" % (ftp_filepath, filepath, ex) ) + else: + for meeting in set([s.meeting for s in doc.session_set.all()]): + resolve_uploaded_material(meeting=meeting, doc=doc) if prev_rev != doc.rev: e = NewRevisionDocEvent(type="new_revision", doc=doc, rev=doc.rev) diff --git a/ietf/doc/views_search.py b/ietf/doc/views_search.py index 67ff0c2f21..4232d77f6c 100644 --- a/ietf/doc/views_search.py +++ b/ietf/doc/views_search.py @@ -59,7 +59,7 @@ import debug # pyflakes:ignore from ietf.doc.models import ( Document, DocHistory, State, - LastCallDocEvent, NewRevisionDocEvent, IESG_SUBSTATE_TAGS, + NewRevisionDocEvent, IESG_SUBSTATE_TAGS, IESG_BALLOT_ACTIVE_STATES, IESG_STATCHG_CONFLREV_ACTIVE_STATES, IESG_CHARTER_ACTIVE_STATES ) from ietf.doc.fields import select2_id_doc_name_json @@ -74,7 +74,7 @@ from ietf.utils.log import log from ietf.doc.utils_search import prepare_document_table, doc_type, doc_state, doc_type_name, AD_WORKLOAD from ietf.ietfauth.utils import has_role - +from ietf.utils.unicodenormalize import normalize_for_sorting class SearchForm(forms.Form): name = forms.CharField(required=False) @@ -219,7 +219,7 @@ def retrieve_search_results(form, all_types=False): queries.extend([Q(targets_related__source__name__icontains=look_for, targets_related__relationship_id="became_rfc")]) combined_query = reduce(operator.or_, queries) - docs = docs.filter(combined_query).distinct() + docs = docs.filter(combined_query) # rfc/active/old check buttons allowed_draft_states = [] @@ -229,20 +229,23 @@ def retrieve_search_results(form, all_types=False): allowed_draft_states.extend(['repl', 'expired', 'auth-rm', 'ietf-rm']) docs = docs.filter(Q(states__slug__in=allowed_draft_states) | - ~Q(type__slug='draft')).distinct() + ~Q(type__slug='draft')) # radio choices by = query["by"] if by == "author": docs = docs.filter( Q(documentauthor__person__alias__name__icontains=query["author"]) | - Q(documentauthor__person__email__address__icontains=query["author"]) + Q(documentauthor__person__email__address__icontains=query["author"]) | + Q(rfcauthor__person__alias__name__icontains=query["author"]) | + Q(rfcauthor__person__email__address__icontains=query["author"]) | + Q(rfcauthor__titlepage_name__icontains=query["author"]) ) elif by == "group": docs = docs.filter(group__acronym__iexact=query["group"]) elif by == "area": docs = docs.filter(Q(group__type="wg", group__parent=query["area"]) | - Q(group=query["area"])).distinct() + Q(group=query["area"])) elif by == "ad": docs = docs.filter(ad=query["ad"]) elif by == "state": @@ -255,6 +258,8 @@ def retrieve_search_results(form, all_types=False): elif by == "stream": docs = docs.filter(stream=query["stream"]) + docs=docs.distinct() + return docs @@ -480,6 +485,7 @@ def _state_to_doc_type(state): ).distinct(): if p in get_active_ads(): ads.append(p) + ads.sort(key=lambda p: normalize_for_sorting(p.plain_name())) bucket_template = { dt: {state: [[] for _ in range(days)] for state in STATE_SLUGS[dt].values()} @@ -849,31 +855,6 @@ def drafts_in_last_call(request): 'form':form, 'docs':results, 'meta':meta, 'pages':pages }) -def drafts_in_iesg_process(request): - states = State.objects.filter(type="draft-iesg").exclude(slug__in=('idexists', 'pub', 'dead', 'rfcqueue')) - title = "Documents in IESG process" - - grouped_docs = [] - - for s in states.order_by("order"): - docs = Document.objects.filter(type="draft", states=s).distinct().order_by("time").select_related("ad", "group", "group__parent") - if docs: - if s.slug == "lc": - for d in docs: - e = d.latest_event(LastCallDocEvent, type="sent_last_call") - # If we don't have an event, use an arbitrary date in the past (but not datetime.datetime.min, - # which causes problems with timezone conversions) - d.lc_expires = e.expires if e else datetime.datetime(1950, 1, 1) - docs = list(docs) - docs.sort(key=lambda d: d.lc_expires) - - grouped_docs.append((s, docs)) - - return render(request, 'doc/drafts_in_iesg_process.html', { - "grouped_docs": grouped_docs, - "title": title, - }) - def recent_drafts(request, days=7): slowcache = caches['slowpages'] cache_key = f'recentdraftsview{days}' diff --git a/ietf/doc/views_stats.py b/ietf/doc/views_stats.py index 0bbf0b91c5..028573b338 100644 --- a/ietf/doc/views_stats.py +++ b/ietf/doc/views_stats.py @@ -18,7 +18,7 @@ from ietf.utils.timezone import date_today -epochday = datetime.datetime.utcfromtimestamp(0).date().toordinal() +epochday = datetime.datetime.fromtimestamp(0, datetime.UTC).date().toordinal() def dt(s): @@ -35,13 +35,13 @@ def model_to_timeline_data(model, field='time', **kwargs): assert field in [ f.name for f in model._meta.get_fields() ] objects = ( model.objects.filter(**kwargs) - .annotate(date=TruncDate(field, tzinfo=datetime.timezone.utc)) + .annotate(date=TruncDate(field, tzinfo=datetime.UTC)) .order_by('date') .values('date') .annotate(count=Count('id'))) if objects.exists(): obj_list = list(objects) - today = date_today(datetime.timezone.utc) + today = date_today(datetime.UTC) if not obj_list[-1]['date'] == today: obj_list += [ {'date': today, 'count': 0} ] data = [ ((e['date'].toordinal()-epochday)*1000*60*60*24, e['count']) for e in obj_list ] diff --git a/ietf/group/admin.py b/ietf/group/admin.py index fedec49d85..685c10aeea 100644 --- a/ietf/group/admin.py +++ b/ietf/group/admin.py @@ -26,14 +26,15 @@ MilestoneGroupEvent, GroupExtResource, Appeal, AppealArtifact ) from ietf.name.models import GroupTypeName -from ietf.utils.validators import validate_external_resource_value +from ietf.utils.admin import SaferTabularInline from ietf.utils.response import permission_denied +from ietf.utils.validators import validate_external_resource_value -class RoleInline(admin.TabularInline): +class RoleInline(SaferTabularInline): model = Role raw_id_fields = ["person", "email"] -class GroupURLInline(admin.TabularInline): +class GroupURLInline(SaferTabularInline): model = GroupURL class GroupForm(forms.ModelForm): diff --git a/ietf/group/models.py b/ietf/group/models.py index 608dcc86b9..a7e3c6616e 100644 --- a/ietf/group/models.py +++ b/ietf/group/models.py @@ -111,6 +111,9 @@ def active_wgs(self): def closed_wgs(self): return self.wgs().exclude(state__in=Group.ACTIVE_STATE_IDS) + def areas(self): + return self.get_queryset().filter(type="area") + def with_meetings(self): return self.get_queryset().filter(type__features__has_meetings=True) @@ -491,6 +494,8 @@ def notify_rfceditor_of_group_name_change(sender, instance=None, **kwargs): current = Group.objects.get(pk=instance.pk) except Group.DoesNotExist: return + if current.type_id == "sdo": + return addr = settings.RFC_EDITOR_GROUP_NOTIFICATION_EMAIL if addr and instance.name != current.name: msg = """ diff --git a/ietf/group/serializers.py b/ietf/group/serializers.py new file mode 100644 index 0000000000..db3b37af48 --- /dev/null +++ b/ietf/group/serializers.py @@ -0,0 +1,44 @@ +# Copyright The IETF Trust 2024-2026, All Rights Reserved +"""django-rest-framework serializers""" + +from drf_spectacular.utils import extend_schema_field +from rest_framework import serializers + +from ietf.person.models import Email +from .models import Group, Role + + +class GroupSerializer(serializers.ModelSerializer): + class Meta: + model = Group + fields = ["acronym", "name", "type", "list_email"] + + +class AreaDirectorSerializer(serializers.Serializer): + """Serialize an area director + + Works with Email or Role + """ + + email = serializers.SerializerMethodField() + + @extend_schema_field(serializers.EmailField) + def get_email(self, instance: Email | Role): + if isinstance(instance, Role): + return instance.email.email_address() + return instance.email_address() + + +class AreaSerializer(serializers.ModelSerializer): + ads = serializers.SerializerMethodField() + + class Meta: + model = Group + fields = ["acronym", "name", "ads"] + + @extend_schema_field(AreaDirectorSerializer(many=True)) + def get_ads(self, area: Group): + return AreaDirectorSerializer( + area.ads if area.is_active else Role.objects.none(), + many=True, + ).data diff --git a/ietf/group/tasks.py b/ietf/group/tasks.py index 693aafb385..ada83e80e2 100644 --- a/ietf/group/tasks.py +++ b/ietf/group/tasks.py @@ -9,12 +9,15 @@ from django.conf import settings from django.template.loader import render_to_string +from django.utils import timezone from ietf.doc.storage_utils import store_file +from ietf.liaisons.models import LiaisonStatement from ietf.utils import log +from ietf.utils.test_runner import disable_coverage -from .models import Group -from .utils import fill_in_charter_info, fill_in_wg_drafts, fill_in_wg_roles +from .models import Group, GroupHistory +from .utils import fill_in_charter_info, fill_in_wg_drafts, fill_in_wg_roles, save_group_in_history from .views import extract_last_name, roles @@ -113,3 +116,117 @@ def generate_wg_summary_files_task(): store_file("indexes", "1wg-summary.txt", f, allow_overwrite=True) with summary_by_acronym_file.open("rb") as f: store_file("indexes", "1wg-summary-by-acronym.txt", f, allow_overwrite=True) + +@shared_task +@disable_coverage() +def run_once_adjust_liaison_groups(): # pragma: no cover + log.log("Starting run_once_adjust_liaison_groups") + if all( + [ + Group.objects.filter( + acronym__in=[ + "3gpp-tsg-ct", + "3gpp-tsg-ran-wg1", + "3gpp-tsg-ran-wg4", + "3gpp-tsg-sa", + "3gpp-tsg-sa-wg5", + "3gpp-tsgct", # duplicates 3gpp-tsg-ct above already + "3gpp-tsgct-ct1", # will normalize all acronyms to hyphenated form + "3gpp-tsgct-ct3", # and consistently match the name + "3gpp-tsgct-ct4", # (particularly use of WG) + "3gpp-tsgran", + "3gpp-tsgran-ran2", + "3gpp-tsgsa", # duplicates 3gpp-tsg-sa above + "3gpp-tsgsa-sa2", # will normalize + "3gpp-tsgsa-sa3", + "3gpp-tsgsa-sa4", + "3gpp-tsgt-wg2", + ] + ).count() + == 16, + not Group.objects.filter( + acronym__in=[ + "3gpp-tsg-ran-wg3", + "3gpp-tsg-ct-wg1", + "3gpp-tsg-ct-wg3", + "3gpp-tsg-ct-wg4", + "3gpp-tsg-ran", + "3gpp-tsg-ran-wg2", + "3gpp-tsg-sa-wg2", + "3gpp-tsg-sa-wg3", + "3gpp-tsg-sa-wg4", + "3gpp-tsg-t-wg2", + ] + ).exists(), + Group.objects.filter(acronym="o3gpptsgran3").exists(), + not LiaisonStatement.objects.filter( + to_groups__acronym__in=["3gpp-tsgct", "3gpp-tsgsa"] + ).exists(), + not LiaisonStatement.objects.filter( + from_groups__acronym="3gpp-tsgct" + ).exists(), + LiaisonStatement.objects.filter(from_groups__acronym="3gpp-tsgsa").count() + == 1, + LiaisonStatement.objects.get(from_groups__acronym="3gpp-tsgsa").pk == 1448, + ] + ): + for old_acronym, new_acronym, new_name in ( + ("o3gpptsgran3", "3gpp-tsg-ran-wg3", "3GPP TSG RAN WG3"), + ("3gpp-tsgct-ct1", "3gpp-tsg-ct-wg1", "3GPP TSG CT WG1"), + ("3gpp-tsgct-ct3", "3gpp-tsg-ct-wg3", "3GPP TSG CT WG3"), + ("3gpp-tsgct-ct4", "3gpp-tsg-ct-wg4", "3GPP TSG CT WG4"), + ("3gpp-tsgran", "3gpp-tsg-ran", "3GPP TSG RAN"), + ("3gpp-tsgran-ran2", "3gpp-tsg-ran-wg2", "3GPP TSG RAN WG2"), + ("3gpp-tsgsa-sa2", "3gpp-tsg-sa-wg2", "3GPP TSG SA WG2"), + ("3gpp-tsgsa-sa3", "3gpp-tsg-sa-wg3", "3GPP TSG SA WG3"), + ("3gpp-tsgsa-sa4", "3gpp-tsg-sa-wg4", "3GPP TSG SA WG4"), + ("3gpp-tsgt-wg2", "3gpp-tsg-t-wg2", "3GPP TSG T WG2"), + ): + group = Group.objects.get(acronym=old_acronym) + save_group_in_history(group) + group.time = timezone.now() + group.acronym = new_acronym + group.name = new_name + if old_acronym.startswith("3gpp-tsgct-"): + group.parent = Group.objects.get(acronym="3gpp-tsg-ct") + elif old_acronym.startswith("3gpp-tsgsa-"): + group.parent = Group.objects.get(acronym="3gpp-tsg-sa") + group.save() + group.groupevent_set.create( + time=group.time, + by_id=1, # (System) + type="info_changed", + desc=f"acronym changed from {old_acronym} to {new_acronym}, name set to {new_name}", + ) + + for acronym, new_name in (("3gpp-tsg-ct", "3GPP TSG CT"),): + group = Group.objects.get(acronym=acronym) + save_group_in_history(group) + group.time = timezone.now() + group.name = new_name + group.save() + group.groupevent_set.create( + time=group.time, + by_id=1, # (System) + type="info_changed", + desc=f"name set to {new_name}", + ) + + ls = LiaisonStatement.objects.get(pk=1448) + ls.from_groups.remove(Group.objects.get(acronym="3gpp-tsgsa")) + ls.from_groups.add(Group.objects.get(acronym="3gpp-tsg-sa")) + + # Rewriting history to effectively merge the histories of the duplicate groups + GroupHistory.objects.filter(parent__acronym="3gpp-tsgsa").update( + parent=Group.objects.get(acronym="3gpp-tsg-sa") + ) + GroupHistory.objects.filter(parent__acronym="3gpp-tsgct").update( + parent=Group.objects.get(acronym="3gpp-tsg-ct") + ) + + deleted = Group.objects.filter( + acronym__in=["3gpp-tsgsa", "3gpp-tsgct"] + ).delete() + log.log(f"Deleted Groups: {deleted}") + else: + log.log("* Refusing to continue as preconditions have changed") diff --git a/ietf/group/tests_info.py b/ietf/group/tests_info.py index eb85860ece..34f8500854 100644 --- a/ietf/group/tests_info.py +++ b/ietf/group/tests_info.py @@ -27,7 +27,7 @@ from ietf.community.models import CommunityList from ietf.community.utils import reset_name_contains_index_for_rule -from ietf.doc.factories import WgDraftFactory, IndividualDraftFactory, CharterFactory, BallotDocEventFactory +from ietf.doc.factories import WgDraftFactory, RgDraftFactory, IndividualDraftFactory, CharterFactory, BallotDocEventFactory from ietf.doc.models import Document, DocEvent, State from ietf.doc.storage_utils import retrieve_str from ietf.doc.utils_charter import charter_name_for_group @@ -413,6 +413,7 @@ def test_group_documents(self): self.assertContains(r, draft3.name) for ah in draft3.action_holders.all(): self.assertContains(r, escape(ah.name)) + self.assertContains(r, "Active with the IESG Internet-Draft") # draft3 is pub-req hence should have such a divider self.assertContains(r, 'for 173 days', count=1) # the old_dah should be tagged self.assertContains(r, draft4.name) self.assertNotContains(r, draft5.name) @@ -425,6 +426,25 @@ def test_group_documents(self): q = PyQuery(r.content) self.assertTrue(any([draft2.name in x.attrib['href'] for x in q('table td a.track-untrack-doc')])) + # Let's also check the IRTF stream + rg = GroupFactory(type_id='rg') + setup_default_community_list_for_group(rg) + rgDraft = RgDraftFactory(group=rg) + rgDraft4 = RgDraftFactory(group=rg) + rgDraft4.set_state(State.objects.get(slug='irsg-w')) + rgDraft7 = RgDraftFactory(group=rg) + rgDraft7.set_state(State.objects.get(type='draft-stream-%s' % rgDraft7.stream_id, slug='dead')) + for url in group_urlreverse_list(rg, 'ietf.group.views.group_documents'): + with self.settings(DOC_ACTION_HOLDER_MAX_AGE_DAYS=20): + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + self.assertContains(r, rgDraft.name) + self.assertContains(r, rg.name) + self.assertContains(r, rg.acronym) + self.assertNotContains(r, draft3.name) # As draft3 is a WG draft, it should not be listed here + self.assertContains(r, rgDraft4.name) + self.assertNotContains(r, rgDraft7.name) + # test the txt version too while we're at it for url in group_urlreverse_list(group, 'ietf.group.views.group_documents_txt'): r = self.client.get(url) diff --git a/ietf/group/tests_serializers.py b/ietf/group/tests_serializers.py new file mode 100644 index 0000000000..bf29e6c8fd --- /dev/null +++ b/ietf/group/tests_serializers.py @@ -0,0 +1,90 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +from ietf.group.factories import RoleFactory, GroupFactory +from ietf.group.serializers import ( + AreaDirectorSerializer, + AreaSerializer, + GroupSerializer, +) +from ietf.person.factories import EmailFactory +from ietf.utils.test_utils import TestCase + + +class GroupSerializerTests(TestCase): + def test_serializes(self): + wg = GroupFactory() + serialized = GroupSerializer(wg).data + self.assertEqual( + serialized, + { + "acronym": wg.acronym, + "name": wg.name, + "type": "wg", + "list_email": wg.list_email, + }, + ) + + +class AreaDirectorSerializerTests(TestCase): + def test_serializes_role(self): + """Should serialize a Role correctly""" + role = RoleFactory(group__type_id="area", name_id="ad") + serialized = AreaDirectorSerializer(role).data + self.assertEqual( + serialized, + {"email": role.email.email_address()}, + ) + + def test_serializes_email(self): + """Should serialize an Email correctly""" + email = EmailFactory() + serialized = AreaDirectorSerializer(email).data + self.assertEqual( + serialized, + {"email": email.email_address()}, + ) + + +class AreaSerializerTests(TestCase): + def test_serializes_active_area(self): + """Should serialize an active area correctly""" + area = GroupFactory(type_id="area", state_id="active") + serialized = AreaSerializer(area).data + self.assertEqual( + serialized, + { + "acronym": area.acronym, + "name": area.name, + "ads": [], + }, + ) + ad_roles = RoleFactory.create_batch(2, group=area, name_id="ad") + serialized = AreaSerializer(area).data + self.assertEqual(serialized["acronym"], area.acronym) + self.assertEqual(serialized["name"], area.name) + self.assertCountEqual( + serialized["ads"], + [{"email": ad.email.email_address()} for ad in ad_roles], + ) + + def test_serializes_inactive_area(self): + """Should serialize an inactive area correctly""" + area = GroupFactory(type_id="area", state_id="conclude") + serialized = AreaSerializer(area).data + self.assertEqual( + serialized, + { + "acronym": area.acronym, + "name": area.name, + "ads": [], + }, + ) + RoleFactory.create_batch(2, group=area, name_id="ad") + serialized = AreaSerializer(area).data + self.assertEqual( + serialized, + { + "acronym": area.acronym, + "name": area.name, + "ads": [], + }, + ) diff --git a/ietf/group/utils.py b/ietf/group/utils.py index 29cfff2c2d..6777ed1933 100644 --- a/ietf/group/utils.py +++ b/ietf/group/utils.py @@ -237,9 +237,10 @@ def construct_group_menu_context(request, group, selected, group_type, others): entries.append(("Review requests", urlreverse(ietf.group.views.review_requests, kwargs=kwargs))) entries.append(("Reviewers", urlreverse(ietf.group.views.reviewer_overview, kwargs=kwargs))) entries.append(("Reviews History", urlreverse(ietf.group.views.review_requests_history, kwargs=kwargs))) - if group.features.has_meetings: entries.append(("Meetings", urlreverse("ietf.group.views.meetings", kwargs=kwargs))) + if group.acronym in ["iesg"]: + entries.append(("Working Groups", urlreverse("ietf.iesg.views.working_groups"))) if group.acronym in ["iab", "iesg"]: entries.append(("Statements", urlreverse("ietf.group.views.statements", kwargs=kwargs))) entries.append(("Appeals", urlreverse("ietf.group.views.appeals", kwargs=kwargs))) @@ -250,7 +251,6 @@ def construct_group_menu_context(request, group, selected, group_type, others): if is_valid_url(group.list_archive): entries.append((mark_safe("List archive »"), group.list_archive)) - # actions actions = [] diff --git a/ietf/group/views.py b/ietf/group/views.py index 3529b31f68..efe3eca15d 100644 --- a/ietf/group/views.py +++ b/ietf/group/views.py @@ -443,7 +443,6 @@ def prepare_group_documents(request, group, clist): return docs, meta, docs_related, meta_related - def get_leadership(group_type): people = Person.objects.filter( role__name__slug="chair", @@ -941,7 +940,7 @@ def meetings(request, acronym, group_type=None): cutoff_date = revsub_dates_by_meeting[s.meeting.pk] else: cutoff_date = s.meeting.date + datetime.timedelta(days=s.meeting.submission_correction_day_offset) - s.cached_is_cutoff = date_today(datetime.timezone.utc) > cutoff_date + s.cached_is_cutoff = date_today(datetime.UTC) > cutoff_date future, in_progress, recent, past = group_sessions(sessions) diff --git a/ietf/idindex/index.py b/ietf/idindex/index.py index 4f021c0dc7..19eb29d4da 100644 --- a/ietf/idindex/index.py +++ b/ietf/idindex/index.py @@ -276,7 +276,7 @@ def active_drafts_index_by_group(extra_values=()): groups = [g for g in groups_dict.values() if hasattr(g, "active_drafts")] groups.sort(key=lambda g: g.acronym) - fallback_time = datetime.datetime(1950, 1, 1, tzinfo=datetime.timezone.utc) + fallback_time = datetime.datetime(1950, 1, 1, tzinfo=datetime.UTC) for g in groups: g.active_drafts.sort(key=lambda d: d.get("initial_rev_time", fallback_time)) @@ -302,6 +302,6 @@ def id_index_txt(with_abstracts=False): return render_to_string("idindex/id_index.txt", { 'groups': groups, - 'time': timezone.now().astimezone(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S %Z"), + 'time': timezone.now().astimezone(datetime.UTC).strftime("%Y-%m-%d %H:%M:%S %Z"), 'with_abstracts': with_abstracts, }) diff --git a/ietf/iesg/agenda.py b/ietf/iesg/agenda.py index 587713089f..ace4c9ec40 100644 --- a/ietf/iesg/agenda.py +++ b/ietf/iesg/agenda.py @@ -133,7 +133,7 @@ def agenda_sections(): ('4.2', {'title':"WG rechartering"}), ('4.2.1', {'title':"Under evaluation for IETF review", 'docs':[]}), ('4.2.2', {'title':"Proposed for approval", 'docs':[]}), - ('5', {'title':"IAB news we can use"}), + ('5', {'title':"IESG Liaison News"}), ('6', {'title':"Management issues"}), ('7', {'title':"Any Other Business (WG News, New Proposals, etc.)"}), ]) diff --git a/ietf/iesg/tests.py b/ietf/iesg/tests.py index 746ea3f56f..e5fbe5da7b 100644 --- a/ietf/iesg/tests.py +++ b/ietf/iesg/tests.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- +from collections import Counter import datetime import io import tarfile @@ -24,7 +25,9 @@ from ietf.group.models import Group, GroupMilestone, Role from ietf.iesg.agenda import get_agenda_date, agenda_data, fill_in_agenda_administrivia, agenda_sections from ietf.iesg.models import TelechatDate, TelechatAgendaContent +from ietf.iesg.utils import get_wg_dashboard_info from ietf.name.models import StreamName, TelechatAgendaSectionName +from ietf.person.factories import PersonFactory from ietf.person.models import Person from ietf.utils.test_utils import TestCase, login_testing_unauthorized, unicontent from ietf.iesg.factories import IESGMgmtItemFactory, TelechatAgendaContentFactory @@ -182,6 +185,1587 @@ def test_ietf_activity(self): r = self.client.get(url) self.assertEqual(r.status_code, 200) + def test_working_groups(self): + # Clean away the wasted built-for-every-test noise + Group.objects.filter(type__in=["wg", "area"]).delete() + + ( + area_summary, + area_totals, + ad_summary, + noad_summary, + ad_totals, + noad_totals, + totals, + wg_summary, + ) = get_wg_dashboard_info() + self.assertEqual(area_summary, []) + self.assertEqual( + area_totals, {"group_count": 0, "doc_count": 0, "page_count": 0} + ) + self.assertEqual(ad_summary, []) + self.assertEqual(noad_summary, []) + self.assertEqual( + ad_totals, + { + "ad_group_count": 0, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + }, + ) + self.assertEqual( + noad_totals, + { + "ad_group_count": 0, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + }, + ) + self.assertEqual( + totals, + { + "group_count": 0, + "doc_count": 0, + "page_count": 0, + "groups_with_docs_count": 0, + }, + ) + self.assertEqual(wg_summary, []) + + # Construct Areas with WGs similar in shape to a real moment of the IETF + + # Note that this test construciton uses the first letter of the wg acronyms + # for convenience to switch on whether groups have documents with assigned ADs. + # (Search for ` if wg_acronym[0] > "g"`) + # There's no other significance to the names of the area directors or the + # acronyms of the areas and groups other than being distinct. Taking the + # values from sets of similar things hopefully helps with debugging the tests. + + areas = {} + for area_acronym in ["red", "orange", "yellow", "green", "blue", "violet"]: + areas[area_acronym] = GroupFactory(type_id="area", acronym=area_acronym) + for ad, area, wgs in [ + ("Alpha", "red", ["bassoon"]), + ("Bravo", "orange", ["celesta"]), + ("Charlie", "orange", ["clarinet", "cymbals"]), + ("Delta", "yellow", ["flute"]), + ("Echo", "yellow", ["glockenspiel"]), + ("Foxtrot", "green", ["gong", "guitar"]), + ("Golf", "green", ["harp"]), + ("Hotel", "blue", ["harpsichord"]), + ("Indigo", "blue", ["oboe", "organ"]), + ("Juliet", "violet", ["piano"]), + ("Kilo", "violet", ["piccolo"]), + ("Lima", "violet", ["saxophone", "tambourine"]), + ]: + p = Person.objects.filter(name=ad).first() or PersonFactory(name=ad) + RoleFactory(group=areas[area], person=p, name_id="ad") + for wg in wgs: + g = GroupFactory(acronym=wg, type_id="wg", parent=areas[area]) + RoleFactory(group=g, person=p, name_id="ad") + + # Some ADs have out of area groups + g = GroupFactory(acronym="timpani", parent=areas["orange"]) + RoleFactory(group=g, person=Person.objects.get(name="Juliet"), name_id="ad") + + ( + area_summary, + area_totals, + ad_summary, + noad_summary, + ad_totals, + noad_totals, + totals, + wg_summary, + ) = get_wg_dashboard_info() + + # checks for the expected result with area sorted by name + self.assertEqual( + area_summary, + [ + { + "area": "blue", + "groups_in_area": 3, + "groups_with_docs": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "area": "green", + "groups_in_area": 3, + "groups_with_docs": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "area": "orange", + "groups_in_area": 4, + "groups_with_docs": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "area": "red", + "groups_in_area": 1, + "groups_with_docs": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "area": "violet", + "groups_in_area": 4, + "groups_with_docs": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "area": "yellow", + "groups_in_area": 2, + "groups_with_docs": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + ], + ) + self.assertEqual( + area_totals, {"group_count": 0, "doc_count": 0, "page_count": 0} + ) + self.assertEqual( + ad_summary, + [ + { + "ad": "Alpha", + "area": "red", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Bravo", + "area": "orange", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Charlie", + "area": "orange", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Delta", + "area": "yellow", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Echo", + "area": "yellow", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Foxtrot", + "area": "green", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Golf", + "area": "green", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Hotel", + "area": "blue", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Indigo", + "area": "blue", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Juliet", + "area": "orange", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Juliet", + "area": "violet", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Kilo", + "area": "violet", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Lima", + "area": "violet", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + ], + ) + self.assertEqual( + noad_summary, + [ + { + "ad": "Alpha", + "area": "red", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Bravo", + "area": "orange", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Charlie", + "area": "orange", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Delta", + "area": "yellow", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Echo", + "area": "yellow", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Foxtrot", + "area": "green", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Golf", + "area": "green", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Hotel", + "area": "blue", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Indigo", + "area": "blue", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Juliet", + "area": "orange", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Juliet", + "area": "violet", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Kilo", + "area": "violet", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + { + "ad": "Lima", + "area": "violet", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0, + "doc_percent": 0, + "page_percent": 0, + }, + ], + ) + self.assertEqual( + ad_totals, + { + "ad_group_count": 17, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + }, + ) + self.assertEqual( + noad_totals, + { + "ad_group_count": 17, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + }, + ) + self.assertEqual( + totals, + { + "group_count": 17, + "doc_count": 0, + "page_count": 0, + "groups_with_docs_count": 0, + }, + ) + self.assertEqual( + wg_summary, + [ + { + "wg": "bassoon", + "area": "red", + "ad": "Alpha", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "celesta", + "area": "orange", + "ad": "Bravo", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "clarinet", + "area": "orange", + "ad": "Charlie", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "cymbals", + "area": "orange", + "ad": "Charlie", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "flute", + "area": "yellow", + "ad": "Delta", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "glockenspiel", + "area": "yellow", + "ad": "Echo", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "gong", + "area": "green", + "ad": "Foxtrot", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "guitar", + "area": "green", + "ad": "Foxtrot", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "harp", + "area": "green", + "ad": "Golf", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "harpsichord", + "area": "blue", + "ad": "Hotel", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "oboe", + "area": "blue", + "ad": "Indigo", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "organ", + "area": "blue", + "ad": "Indigo", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "piano", + "area": "violet", + "ad": "Juliet", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "piccolo", + "area": "violet", + "ad": "Kilo", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "saxophone", + "area": "violet", + "ad": "Lima", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "tambourine", + "area": "violet", + "ad": "Lima", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "timpani", + "area": "orange", + "ad": "Juliet", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + ], + ) + + # As seen above, all doc and page counts are currently 0 + + # We'll give a group a document but not assign it to its AD + WgDraftFactory( + group=Group.objects.get(acronym="saxophone"), pages=len("saxophone") + ) + ( + area_summary, + area_totals, + ad_summary, + noad_summary, + ad_totals, + noad_totals, + totals, + wg_summary, + ) = get_wg_dashboard_info() + count_violet_dicts = 0 + for d in area_summary: + if d["area"] == "violet": + count_violet_dicts += 1 + self.assertEqual(d["groups_with_docs"], 1) + self.assertEqual(d["doc_count"], 1) + self.assertEqual(d["page_count"], 9) + self.assertEqual(d["group_percent"], 100.0) + self.assertEqual(d["doc_percent"], 100.0) + self.assertEqual(d["page_percent"], 100.0) + else: + self.assertEqual(d["groups_with_docs"], 0) + self.assertEqual(d["doc_count"], 0) + self.assertEqual(d["page_count"], 0) + self.assertEqual(d["group_percent"], 0) + self.assertEqual(d["doc_percent"], 0) + self.assertEqual(d["page_percent"], 0) + self.assertEqual(count_violet_dicts, 1) + + self.assertEqual( + area_totals, {"group_count": 1, "doc_count": 1, "page_count": 9} + ) + + # No AD has this document, even though it's in Lima's group + count_lima_dicts = 0 + for d in ad_summary: + if d["ad"] == "Lima": + count_lima_dicts += 1 + self.assertEqual(d["doc_group_count"], 0) + self.assertEqual(d["doc_count"], 0) + self.assertEqual(d["page_count"], 0) + self.assertEqual(d["group_percent"], 0) + self.assertEqual(d["doc_percent"], 0) + self.assertEqual(d["page_percent"], 0) + self.assertEqual(count_lima_dicts, 1) + + # It's in Lima's group, so normally it will eventually land on Lima + count_lima_dicts = 0 + for d in noad_summary: + if d["ad"] == "Lima": + count_lima_dicts += 1 + self.assertEqual(d["doc_group_count"], 1) + self.assertEqual(d["doc_count"], 1) + self.assertEqual(d["page_count"], 9) + self.assertEqual(d["group_percent"], 100.0) + self.assertEqual(d["doc_percent"], 100.0) + self.assertEqual(d["page_percent"], 100.0) + else: + self.assertEqual(d["doc_group_count"], 0) + self.assertEqual(d["doc_count"], 0) + self.assertEqual(d["page_count"], 0) + self.assertEqual(d["group_percent"], 0) + self.assertEqual(d["doc_percent"], 0) + self.assertEqual(d["page_percent"], 0) + self.assertEqual(count_lima_dicts, 1) + + self.assertEqual( + ad_totals, + { + "ad_group_count": 17, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + }, + ) + self.assertEqual( + noad_totals, + { + "ad_group_count": 17, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 9, + }, + ) + self.assertEqual( + totals, + { + "group_count": 17, + "doc_count": 1, + "page_count": 9, + "groups_with_docs_count": 1, + }, + ) + + count_sax_dicts = 0 + for d in wg_summary: + if d["wg"] == "saxophone": + count_sax_dicts += 1 + self.assertEqual(d["doc_count"], 1) + self.assertEqual(d["page_count"], 9) + else: + self.assertEqual(d["doc_count"], 0) + self.assertEqual(d["page_count"], 0) + self.assertEqual(count_sax_dicts, 1) + + # Assign that doc to Lima + self.assertEqual(Document.objects.count(), 1) + Document.objects.all().update(ad=Person.objects.get(name="Lima")) + ( + area_summary, + area_totals, + ad_summary, + noad_summary, + ad_totals, + noad_totals, + totals, + wg_summary, + ) = get_wg_dashboard_info() + count_violet_dicts = 0 + for d in area_summary: + if d["area"] == "violet": + count_violet_dicts += 1 + self.assertEqual(d["groups_with_docs"], 1) + self.assertEqual(d["doc_count"], 1) + self.assertEqual(d["page_count"], 9) + self.assertEqual(d["group_percent"], 100.0) + self.assertEqual(d["doc_percent"], 100.0) + self.assertEqual(d["page_percent"], 100.0) + else: + self.assertEqual(d["groups_with_docs"], 0) + self.assertEqual(d["doc_count"], 0) + self.assertEqual(d["page_count"], 0) + self.assertEqual(d["group_percent"], 0) + self.assertEqual(d["doc_percent"], 0) + self.assertEqual(d["page_percent"], 0) + self.assertEqual(count_violet_dicts, 1) + + self.assertEqual( + area_totals, {"group_count": 1, "doc_count": 1, "page_count": 9} + ) + + # This time it will show up as a doc assigned to Lima + count_lima_dicts = 0 + for d in ad_summary: + if d["ad"] == "Lima": + count_lima_dicts += 1 + self.assertEqual(d["doc_group_count"], 1) + self.assertEqual(d["doc_count"], 1) + self.assertEqual(d["page_count"], 9) + self.assertEqual(d["group_percent"], 100.0) + self.assertEqual(d["doc_percent"], 100.0) + self.assertEqual(d["page_percent"], 100.0) + else: + self.assertEqual(d["doc_group_count"], 0) + self.assertEqual(d["doc_count"], 0) + self.assertEqual(d["page_count"], 0) + self.assertEqual(d["group_percent"], 0) + self.assertEqual(d["doc_percent"], 0) + self.assertEqual(d["page_percent"], 0) + self.assertEqual(count_lima_dicts, 1) + + # and there will be no noad documents + count_lima_dicts = 0 + for d in noad_summary: + if d["ad"] == "Lima": + count_lima_dicts += 1 + self.assertEqual(d["doc_group_count"], 0) + self.assertEqual(d["doc_count"], 0) + self.assertEqual(d["page_count"], 0) + self.assertEqual(d["group_percent"], 0) + self.assertEqual(d["doc_percent"], 0) + self.assertEqual(d["page_percent"], 0) + self.assertEqual(count_lima_dicts, 1) + + self.assertEqual( + ad_totals, + { + "ad_group_count": 17, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 9, + }, + ) + self.assertEqual( + noad_totals, + { + "ad_group_count": 17, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + }, + ) + self.assertEqual( + totals, + { + "group_count": 17, + "doc_count": 1, + "page_count": 9, + "groups_with_docs_count": 1, + }, + ) + + count_sax_dicts = 0 + for d in wg_summary: + if d["wg"] == "saxophone": + count_sax_dicts += 1 + self.assertEqual(d["doc_count"], 1) + self.assertEqual(d["page_count"], 9) + else: + self.assertEqual(d["doc_count"], 0) + self.assertEqual(d["page_count"], 0) + self.assertEqual(count_sax_dicts, 1) + + # Now give Lima a document in a group that's not in their area: + WgDraftFactory( + group=Group.objects.get(acronym="gong"), + pages=len("gong"), + ad=Person.objects.get(name="Lima"), + ) + ( + area_summary, + area_totals, + ad_summary, + noad_summary, + ad_totals, + noad_totals, + totals, + wg_summary, + ) = get_wg_dashboard_info() + seen_dicts = Counter([d["area"] for d in area_summary]) + for d in areas: + self.assertEqual(seen_dicts[area], 1 if area in ["violet", "green"] else 0) + for d in area_summary: + if d["area"] in ["violet", "green"]: + self.assertEqual(d["doc_count"], 1) + self.assertEqual(d["page_count"], 9 if d["area"] == "violet" else 4) + self.assertEqual(d["group_percent"], 50) + self.assertEqual(d["doc_percent"], 50) + self.assertEqual( + d["page_percent"], + 100 * 9 / 13 if d["area"] == "violet" else 100 * 4 / 13, + ) + else: + self.assertEqual(d["doc_count"], 0) + self.assertEqual(d["page_count"], 0) + self.assertEqual(d["group_percent"], 0) + self.assertEqual(d["doc_percent"], 0) + self.assertEqual(d["page_percent"], 0) + + self.assertEqual( + area_totals, {"group_count": 2, "doc_count": 2, "page_count": 13} + ) + + for d in ad_summary: + if d["ad"] == "Lima": + self.assertEqual(d["doc_group_count"], 1) + self.assertEqual(d["doc_count"], 1) + self.assertEqual(d["page_count"], 9 if d["area"] == "violet" else 4) + self.assertEqual(d["group_percent"], 50) + self.assertEqual(d["doc_percent"], 50) + self.assertEqual( + d["page_percent"], + 100 * 9 / 13 if d["area"] == "violet" else 100 * 4 / 13, + ) + else: + self.assertEqual(d["doc_group_count"], 0) + self.assertEqual( + d["doc_count"], 0 + ) # Note in particular this is 0 for Foxtrot + self.assertEqual(d["page_count"], 0) + self.assertEqual(d["group_percent"], 0) + self.assertEqual(d["doc_percent"], 0) + self.assertEqual(d["page_percent"], 0) + + for d in wg_summary: + if d["wg"] == "gong": + # Lima's doc in gong above counts at the dict for gong even though the ad reported there is Foxtrot. + self.assertEqual( + d, + { + "wg": "gong", + "area": "green", + "ad": "Foxtrot", + "doc_count": 1, + "page_count": 4, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + ) + elif d["ad"] == "Lima": + self.assertEqual( + d["area"], "violet" + ) # The out of area assignment is not reflected in the wg_summary at all. + + # Now pile on a lot of documents + for wg_acronym in [ + "bassoon", + "celesta", + "clarinet", + "cymbals", + "flute", + "glockenspiel", + "gong", + "guitar", + "harp", + "harpsichord", + "oboe", + "organ", + "piano", + "piccolo", + "saxophone", + "tambourine", + "timpani", + ]: + if wg_acronym in ["bassoon", "celesta"]: + continue # Those WGs have no docs + # The rest have a doc that's not assigned to any ad + WgDraftFactory( + group=Group.objects.get(acronym=wg_acronym), pages=len(wg_acronym) + ) + if wg_acronym[0] > "g": + # Some have a doc assigned to the responsible ad + WgDraftFactory( + group=Group.objects.get(acronym=wg_acronym), + pages=len(wg_acronym), + ad=Role.objects.get(name_id="ad", group__acronym=wg_acronym).person, + ) + # The other AD for an area might be covering a doc + WgDraftFactory( + group=Group.objects.get(acronym="saxophone"), + pages=len("saxophone"), + ad=Person.objects.get(name="Juliet"), + ) + # An Ad not associated with the group or the area is responsible for a doc + WgDraftFactory( + group=Group.objects.get(acronym="bassoon"), + pages=len("bassoon"), + ad=Person.objects.get(name="Juliet"), + ) + + ( + area_summary, + area_totals, + ad_summary, + noad_summary, + ad_totals, + noad_totals, + totals, + wg_summary, + ) = get_wg_dashboard_info() + + self.assertEqual( + area_summary, + [ + { + "area": "blue", + "groups_in_area": 3, + "groups_with_docs": 3, + "doc_count": 6, + "page_count": 40, + "group_percent": 18.75, + "doc_percent": 21.428571428571427, + "page_percent": 20.51282051282051, + }, + { + "area": "green", + "groups_in_area": 3, + "groups_with_docs": 3, + "doc_count": 5, + "page_count": 22, + "group_percent": 18.75, + "doc_percent": 17.857142857142858, + "page_percent": 11.282051282051283, + }, + { + "area": "orange", + "groups_in_area": 4, + "groups_with_docs": 3, + "doc_count": 4, + "page_count": 29, + "group_percent": 18.75, + "doc_percent": 14.285714285714285, + "page_percent": 14.871794871794872, + }, + { + "area": "red", + "groups_in_area": 1, + "groups_with_docs": 1, + "doc_count": 1, + "page_count": 7, + "group_percent": 6.25, + "doc_percent": 3.571428571428571, + "page_percent": 3.5897435897435894, + }, + { + "area": "violet", + "groups_in_area": 4, + "groups_with_docs": 4, + "doc_count": 10, + "page_count": 80, + "group_percent": 25.0, + "doc_percent": 35.714285714285715, + "page_percent": 41.02564102564102, + }, + { + "area": "yellow", + "groups_in_area": 2, + "groups_with_docs": 2, + "doc_count": 2, + "page_count": 17, + "group_percent": 12.5, + "doc_percent": 7.142857142857142, + "page_percent": 8.717948717948717, + }, + ], + ) + self.assertEqual( + area_totals, {"group_count": 16, "doc_count": 28, "page_count": 195} + ) + self.assertEqual( + ad_summary, + [ + { + "ad": "Alpha", + "area": "red", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0.0, + "doc_percent": 0.0, + "page_percent": 0.0, + }, + { + "ad": "Bravo", + "area": "orange", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0.0, + "doc_percent": 0.0, + "page_percent": 0.0, + }, + { + "ad": "Charlie", + "area": "orange", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0.0, + "doc_percent": 0.0, + "page_percent": 0.0, + }, + { + "ad": "Delta", + "area": "yellow", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0.0, + "doc_percent": 0.0, + "page_percent": 0.0, + }, + { + "ad": "Echo", + "area": "yellow", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0.0, + "doc_percent": 0.0, + "page_percent": 0.0, + }, + { + "ad": "Foxtrot", + "area": "green", + "ad_group_count": 2, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0.0, + "doc_percent": 0.0, + "page_percent": 0.0, + }, + { + "ad": "Golf", + "area": "green", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 4, + "group_percent": 8.333333333333332, + "doc_percent": 7.6923076923076925, + "page_percent": 4.395604395604396, + }, + { + "ad": "Hotel", + "area": "blue", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 11, + "group_percent": 8.333333333333332, + "doc_percent": 7.6923076923076925, + "page_percent": 12.087912087912088, + }, + { + "ad": "Indigo", + "area": "blue", + "ad_group_count": 2, + "doc_group_count": 2, + "doc_count": 2, + "page_count": 9, + "group_percent": 16.666666666666664, + "doc_percent": 15.384615384615385, + "page_percent": 9.89010989010989, + }, + { + "ad": "Juliet", + "area": "orange", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 7, + "group_percent": 8.333333333333332, + "doc_percent": 7.6923076923076925, + "page_percent": 7.6923076923076925, + }, + { + "ad": "Juliet", + "area": "red", + "ad_group_count": 0, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 7, + "group_percent": 8.333333333333332, + "doc_percent": 7.6923076923076925, + "page_percent": 7.6923076923076925, + }, + { + "ad": "Juliet", + "area": "violet", + "ad_group_count": 1, + "doc_group_count": 2, + "doc_count": 2, + "page_count": 14, + "group_percent": 16.666666666666664, + "doc_percent": 15.384615384615385, + "page_percent": 15.384615384615385, + }, + { + "ad": "Kilo", + "area": "violet", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 7, + "group_percent": 8.333333333333332, + "doc_percent": 7.6923076923076925, + "page_percent": 7.6923076923076925, + }, + { + "ad": "Lima", + "area": "green", + "ad_group_count": 0, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 4, + "group_percent": 8.333333333333332, + "doc_percent": 7.6923076923076925, + "page_percent": 4.395604395604396, + }, + { + "ad": "Lima", + "area": "violet", + "ad_group_count": 2, + "doc_group_count": 2, + "doc_count": 3, + "page_count": 28, + "group_percent": 16.666666666666664, + "doc_percent": 23.076923076923077, + "page_percent": 30.76923076923077, + }, + ], + ) + self.assertEqual( + noad_summary, + [ + { + "ad": "Alpha", + "area": "red", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0.0, + "doc_percent": 0.0, + "page_percent": 0.0, + }, + { + "ad": "Bravo", + "area": "orange", + "ad_group_count": 1, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + "group_percent": 0.0, + "doc_percent": 0.0, + "page_percent": 0.0, + }, + { + "ad": "Charlie", + "area": "orange", + "ad_group_count": 2, + "doc_group_count": 2, + "doc_count": 2, + "page_count": 15, + "group_percent": 13.333333333333334, + "doc_percent": 13.333333333333334, + "page_percent": 14.423076923076922, + }, + { + "ad": "Delta", + "area": "yellow", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 5, + "group_percent": 6.666666666666667, + "doc_percent": 6.666666666666667, + "page_percent": 4.807692307692308, + }, + { + "ad": "Echo", + "area": "yellow", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 12, + "group_percent": 6.666666666666667, + "doc_percent": 6.666666666666667, + "page_percent": 11.538461538461538, + }, + { + "ad": "Foxtrot", + "area": "green", + "ad_group_count": 2, + "doc_group_count": 2, + "doc_count": 2, + "page_count": 10, + "group_percent": 13.333333333333334, + "doc_percent": 13.333333333333334, + "page_percent": 9.615384615384617, + }, + { + "ad": "Golf", + "area": "green", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 4, + "group_percent": 6.666666666666667, + "doc_percent": 6.666666666666667, + "page_percent": 3.8461538461538463, + }, + { + "ad": "Hotel", + "area": "blue", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 11, + "group_percent": 6.666666666666667, + "doc_percent": 6.666666666666667, + "page_percent": 10.576923076923077, + }, + { + "ad": "Indigo", + "area": "blue", + "ad_group_count": 2, + "doc_group_count": 2, + "doc_count": 2, + "page_count": 9, + "group_percent": 13.333333333333334, + "doc_percent": 13.333333333333334, + "page_percent": 8.653846153846153, + }, + { + "ad": "Juliet", + "area": "orange", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 7, + "group_percent": 6.666666666666667, + "doc_percent": 6.666666666666667, + "page_percent": 6.730769230769231, + }, + { + "ad": "Juliet", + "area": "violet", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 5, + "group_percent": 6.666666666666667, + "doc_percent": 6.666666666666667, + "page_percent": 4.807692307692308, + }, + { + "ad": "Kilo", + "area": "violet", + "ad_group_count": 1, + "doc_group_count": 1, + "doc_count": 1, + "page_count": 7, + "group_percent": 6.666666666666667, + "doc_percent": 6.666666666666667, + "page_percent": 6.730769230769231, + }, + { + "ad": "Lima", + "area": "violet", + "ad_group_count": 2, + "doc_group_count": 2, + "doc_count": 2, + "page_count": 19, + "group_percent": 13.333333333333334, + "doc_percent": 13.333333333333334, + "page_percent": 18.269230769230766, + }, + ], + ) + self.assertEqual( + ad_totals, + { + "ad_group_count": 17, + "doc_group_count": 12, + "doc_count": 13, + "page_count": 91, + }, + ) + self.assertEqual( + noad_totals, + { + "ad_group_count": 17, + "doc_group_count": 15, + "doc_count": 15, + "page_count": 104, + }, + ) + self.assertEqual( + totals, + { + "group_count": 17, + "doc_count": 28, + "page_count": 195, + "groups_with_docs_count": 16, + }, + ) + self.assertEqual( + wg_summary, + [ + { + "wg": "bassoon", + "area": "red", + "ad": "Alpha", + "doc_count": 1, + "page_count": 7, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "celesta", + "area": "orange", + "ad": "Bravo", + "doc_count": 0, + "page_count": 0, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "clarinet", + "area": "orange", + "ad": "Charlie", + "doc_count": 1, + "page_count": 8, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "cymbals", + "area": "orange", + "ad": "Charlie", + "doc_count": 1, + "page_count": 7, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "flute", + "area": "yellow", + "ad": "Delta", + "doc_count": 1, + "page_count": 5, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "glockenspiel", + "area": "yellow", + "ad": "Echo", + "doc_count": 1, + "page_count": 12, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "gong", + "area": "green", + "ad": "Foxtrot", + "doc_count": 2, + "page_count": 8, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "guitar", + "area": "green", + "ad": "Foxtrot", + "doc_count": 1, + "page_count": 6, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "harp", + "area": "green", + "ad": "Golf", + "doc_count": 2, + "page_count": 8, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "harpsichord", + "area": "blue", + "ad": "Hotel", + "doc_count": 2, + "page_count": 22, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "oboe", + "area": "blue", + "ad": "Indigo", + "doc_count": 2, + "page_count": 8, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "organ", + "area": "blue", + "ad": "Indigo", + "doc_count": 2, + "page_count": 10, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "piano", + "area": "violet", + "ad": "Juliet", + "doc_count": 2, + "page_count": 10, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "piccolo", + "area": "violet", + "ad": "Kilo", + "doc_count": 2, + "page_count": 14, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "saxophone", + "area": "violet", + "ad": "Lima", + "doc_count": 4, + "page_count": 36, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "tambourine", + "area": "violet", + "ad": "Lima", + "doc_count": 2, + "page_count": 20, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + { + "wg": "timpani", + "area": "orange", + "ad": "Juliet", + "doc_count": 2, + "page_count": 14, + "rfc_count": 0, + "recent_rfc_count": 0, + }, + ], + ) + + # Make sure the view doesn't _crash_ - the template is a dead-simple rendering of the dicts, but this test doesn't prove that + url = urlreverse("ietf.iesg.views.working_groups") + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + class IESGAgendaTests(TestCase): def setUp(self): diff --git a/ietf/iesg/urls.py b/ietf/iesg/urls.py index d8cfec9f90..5fd9dea0cc 100644 --- a/ietf/iesg/urls.py +++ b/ietf/iesg/urls.py @@ -59,6 +59,7 @@ url(r'^agenda/telechat-(?:%(date)s-)?docs.tgz' % settings.URL_REGEXPS, views.telechat_docs_tarfile), url(r'^discusses/$', views.discusses), url(r'^ietf-activity/$', views.ietf_activity), + url(r'^working-groups/$', views.working_groups), url(r'^milestones/$', views.milestones_needing_review), url(r'^photos/$', views.photos), ] diff --git a/ietf/iesg/utils.py b/ietf/iesg/utils.py index 56571dc753..1d24ecac8e 100644 --- a/ietf/iesg/utils.py +++ b/ietf/iesg/utils.py @@ -1,32 +1,45 @@ -from collections import namedtuple +from collections import Counter, defaultdict, namedtuple -import debug # pyflakes:ignore +import datetime + +import debug # pyflakes:ignore + +from django.db import models +from django.utils import timezone from ietf.doc.models import Document, STATUSCHANGE_RELATIONS from ietf.doc.utils_search import fill_in_telechat_date +from ietf.group.models import Group from ietf.iesg.agenda import get_doc_section +from ietf.person.utils import get_active_ads +from ietf.utils.unicodenormalize import normalize_for_sorting +TelechatPageCount = namedtuple( + "TelechatPageCount", + ["for_approval", "for_action", "related", "ad_pages_left_to_ballot_on"], +) -TelechatPageCount = namedtuple('TelechatPageCount',['for_approval','for_action','related','ad_pages_left_to_ballot_on']) def telechat_page_count(date=None, docs=None, ad=None): if not date and not docs: return TelechatPageCount(0, 0, 0, 0) if not docs: - candidates = Document.objects.filter(docevent__telechatdocevent__telechat_date=date).distinct() + candidates = Document.objects.filter( + docevent__telechatdocevent__telechat_date=date + ).distinct() fill_in_telechat_date(candidates) - docs = [ doc for doc in candidates if doc.telechat_date()==date ] + docs = [doc for doc in candidates if doc.telechat_date() == date] - for_action =[d for d in docs if get_doc_section(d).endswith('.3')] + for_action = [d for d in docs if get_doc_section(d).endswith(".3")] - for_approval = set(docs)-set(for_action) + for_approval = set(docs) - set(for_action) - drafts = [d for d in for_approval if d.type_id == 'draft'] + drafts = [d for d in for_approval if d.type_id == "draft"] ad_pages_left_to_ballot_on = 0 pages_for_approval = 0 - + for draft in drafts: pages_for_approval += draft.pages or 0 if ad: @@ -39,30 +52,270 @@ def telechat_page_count(date=None, docs=None, ad=None): pages_for_action = 0 for d in for_action: - if d.type_id == 'draft': + if d.type_id == "draft": pages_for_action += d.pages or 0 - elif d.type_id == 'statchg': + elif d.type_id == "statchg": for rel in d.related_that_doc(STATUSCHANGE_RELATIONS): pages_for_action += rel.pages or 0 - elif d.type_id == 'conflrev': - for rel in d.related_that_doc('conflrev'): + elif d.type_id == "conflrev": + for rel in d.related_that_doc("conflrev"): pages_for_action += rel.pages or 0 else: pass related_pages = 0 - for d in for_approval-set(drafts): - if d.type_id == 'statchg': + for d in for_approval - set(drafts): + if d.type_id == "statchg": for rel in d.related_that_doc(STATUSCHANGE_RELATIONS): related_pages += rel.pages or 0 - elif d.type_id == 'conflrev': - for rel in d.related_that_doc('conflrev'): + elif d.type_id == "conflrev": + for rel in d.related_that_doc("conflrev"): related_pages += rel.pages or 0 else: # There's really nothing to rely on to give a reading load estimate for charters pass - - return TelechatPageCount(for_approval=pages_for_approval, - for_action=pages_for_action, - related=related_pages, - ad_pages_left_to_ballot_on=ad_pages_left_to_ballot_on) + + return TelechatPageCount( + for_approval=pages_for_approval, + for_action=pages_for_action, + related=related_pages, + ad_pages_left_to_ballot_on=ad_pages_left_to_ballot_on, + ) + + +def get_wg_dashboard_info(): + docs = ( + Document.objects.filter( + group__type="wg", + group__state="active", + states__type="draft", + states__slug="active", + ) + .filter(models.Q(ad__isnull=True) | models.Q(ad__in=get_active_ads())) + .distinct() + .prefetch_related("group", "group__parent") + .exclude( + states__type="draft-stream-ietf", + states__slug__in=["c-adopt", "wg-cand", "dead", "parked", "info"], + ) + ) + groups = Group.objects.filter(state="active", type="wg") + areas = Group.objects.filter(state="active", type="area") + + total_group_count = groups.count() + total_doc_count = docs.count() + total_page_count = docs.aggregate(models.Sum("pages"))["pages__sum"] or 0 + totals = { + "group_count": total_group_count, + "doc_count": total_doc_count, + "page_count": total_page_count, + } + + # Since this view is primarily about counting subsets of the above docs query and the + # expected number of returned documents is just under 1000 typically - do the totaling + # work in python rather than asking the db to do it. + + groups_for_area = defaultdict(set) + pages_for_area = defaultdict(lambda: 0) + docs_for_area = defaultdict(lambda: 0) + groups_for_ad = defaultdict(lambda: defaultdict(set)) + pages_for_ad = defaultdict(lambda: defaultdict(lambda: 0)) + docs_for_ad = defaultdict(lambda: defaultdict(lambda: 0)) + groups_for_noad = defaultdict(lambda: defaultdict(set)) + pages_for_noad = defaultdict(lambda: defaultdict(lambda: 0)) + docs_for_noad = defaultdict(lambda: defaultdict(lambda: 0)) + docs_for_wg = defaultdict(lambda: 0) + pages_for_wg = defaultdict(lambda: 0) + groups_total = set() + pages_total = 0 + docs_total = 0 + + responsible_for_group = defaultdict(lambda: defaultdict(lambda: "None")) + responsible_count = defaultdict(lambda: defaultdict(lambda: 0)) + for group in groups: + responsible = f"{', '.join([r.person.plain_name() for r in group.role_set.filter(name_id='ad')])}" + docs_for_noad[responsible][group.parent.acronym] = ( + 0 # Ensure these keys are present later + ) + docs_for_ad[responsible][group.parent.acronym] = 0 + responsible_for_group[group.acronym][group.parent.acronym] = responsible + responsible_count[responsible][group.parent.acronym] += 1 + + for doc in docs: + docs_for_wg[doc.group] += 1 + pages_for_wg[doc.group] += doc.pages + groups_for_area[doc.group.area.acronym].add(doc.group.acronym) + pages_for_area[doc.group.area.acronym] += doc.pages + docs_for_area[doc.group.area.acronym] += 1 + + if doc.ad is None: + responsible = responsible_for_group[doc.group.acronym][ + doc.group.parent.acronym + ] + groups_for_noad[responsible][doc.group.parent.acronym].add( + doc.group.acronym + ) + pages_for_noad[responsible][doc.group.parent.acronym] += doc.pages + docs_for_noad[responsible][doc.group.parent.acronym] += 1 + else: + responsible = f"{doc.ad.plain_name()}" + groups_for_ad[responsible][doc.group.parent.acronym].add(doc.group.acronym) + pages_for_ad[responsible][doc.group.parent.acronym] += doc.pages + docs_for_ad[responsible][doc.group.parent.acronym] += 1 + + docs_total += 1 + groups_total.add(doc.group.acronym) + pages_total += doc.pages + + groups_total = len(groups_total) + totals["groups_with_docs_count"] = groups_total + + area_summary = [] + + for area in areas: + group_count = len(groups_for_area[area.acronym]) + doc_count = docs_for_area[area.acronym] + page_count = pages_for_area[area.acronym] + area_summary.append( + { + "area": area.acronym, + "groups_in_area": groups.filter(parent=area).count(), + "groups_with_docs": group_count, + "doc_count": doc_count, + "page_count": page_count, + "group_percent": group_count / groups_total * 100 + if groups_total != 0 + else 0, + "doc_percent": doc_count / docs_total * 100 if docs_total != 0 else 0, + "page_percent": page_count / pages_total * 100 + if pages_total != 0 + else 0, + } + ) + area_summary.sort(key=lambda r: r["area"]) + area_totals = { + "group_count": groups_total, + "doc_count": docs_total, + "page_count": pages_total, + } + + noad_summary = [] + noad_totals = { + "ad_group_count": 0, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + } + for ad in docs_for_noad: + for area in docs_for_noad[ad]: + noad_totals["ad_group_count"] += responsible_count[ad][area] + noad_totals["doc_group_count"] += len(groups_for_noad[ad][area]) + noad_totals["doc_count"] += docs_for_noad[ad][area] + noad_totals["page_count"] += pages_for_noad[ad][area] + for ad in docs_for_noad: + for area in docs_for_noad[ad]: + noad_summary.append( + { + "ad": ad, + "area": area, + "ad_group_count": responsible_count[ad][area], + "doc_group_count": len(groups_for_noad[ad][area]), + "doc_count": docs_for_noad[ad][area], + "page_count": pages_for_noad[ad][area], + "group_percent": len(groups_for_noad[ad][area]) + / noad_totals["doc_group_count"] + * 100 + if noad_totals["doc_group_count"] != 0 + else 0, + "doc_percent": docs_for_noad[ad][area] + / noad_totals["doc_count"] + * 100 + if noad_totals["doc_count"] != 0 + else 0, + "page_percent": pages_for_noad[ad][area] + / noad_totals["page_count"] + * 100 + if noad_totals["page_count"] != 0 + else 0, + } + ) + noad_summary.sort(key=lambda r: (normalize_for_sorting(r["ad"]), r["area"])) + + ad_summary = [] + ad_totals = { + "ad_group_count": 0, + "doc_group_count": 0, + "doc_count": 0, + "page_count": 0, + } + for ad in docs_for_ad: + for area in docs_for_ad[ad]: + ad_totals["ad_group_count"] += responsible_count[ad][area] + ad_totals["doc_group_count"] += len(groups_for_ad[ad][area]) + ad_totals["doc_count"] += docs_for_ad[ad][area] + ad_totals["page_count"] += pages_for_ad[ad][area] + for ad in docs_for_ad: + for area in docs_for_ad[ad]: + ad_summary.append( + { + "ad": ad, + "area": area, + "ad_group_count": responsible_count[ad][area], + "doc_group_count": len(groups_for_ad[ad][area]), + "doc_count": docs_for_ad[ad][area], + "page_count": pages_for_ad[ad][area], + "group_percent": len(groups_for_ad[ad][area]) + / ad_totals["doc_group_count"] + * 100 + if ad_totals["doc_group_count"] != 0 + else 0, + "doc_percent": docs_for_ad[ad][area] / ad_totals["doc_count"] * 100 + if ad_totals["doc_count"] != 0 + else 0, + "page_percent": pages_for_ad[ad][area] + / ad_totals["page_count"] + * 100 + if ad_totals["page_count"] != 0 + else 0, + } + ) + ad_summary.sort(key=lambda r: (normalize_for_sorting(r["ad"]), r["area"])) + + rfc_counter = Counter( + Document.objects.filter(type="rfc").values_list("group__acronym", flat=True) + ) + recent_rfc_counter = Counter( + Document.objects.filter( + type="rfc", + docevent__type="published_rfc", + docevent__time__gte=timezone.now() - datetime.timedelta(weeks=104), + ).values_list("group__acronym", flat=True) + ) + for wg in set(groups) - set(docs_for_wg.keys()): + docs_for_wg[wg] += 0 + pages_for_wg[wg] += 0 + wg_summary = [] + for wg in docs_for_wg: + wg_summary.append( + { + "wg": wg.acronym, + "area": wg.parent.acronym, + "ad": responsible_for_group[wg.acronym][wg.parent.acronym], + "doc_count": docs_for_wg[wg], + "page_count": pages_for_wg[wg], + "rfc_count": rfc_counter[wg.acronym], + "recent_rfc_count": recent_rfc_counter[wg.acronym], + } + ) + wg_summary.sort(key=lambda r: (r["wg"], r["area"])) + + return ( + area_summary, + area_totals, + ad_summary, + noad_summary, + ad_totals, + noad_totals, + totals, + wg_summary, + ) diff --git a/ietf/iesg/views.py b/ietf/iesg/views.py index 7b9f489b44..f03afb9fc1 100644 --- a/ietf/iesg/views.py +++ b/ietf/iesg/views.py @@ -61,13 +61,14 @@ from ietf.group.models import GroupMilestone, Role from ietf.iesg.agenda import agenda_data, agenda_sections, fill_in_agenda_docs, get_agenda_date from ietf.iesg.models import TelechatDate, TelechatAgendaContent -from ietf.iesg.utils import telechat_page_count +from ietf.iesg.utils import get_wg_dashboard_info, telechat_page_count from ietf.ietfauth.utils import has_role, role_required, user_is_person from ietf.name.models import TelechatAgendaSectionName from ietf.person.models import Person from ietf.meeting.utils import get_activity_stats from ietf.doc.utils_search import fill_in_document_table_attributes, fill_in_telechat_date from ietf.utils.timezone import date_today, datetime_from_date +from ietf.utils.unicodenormalize import normalize_for_sorting def review_decisions(request, year=None): events = DocEvent.objects.filter(type__in=("iesg_disapproved", "iesg_approved")) @@ -101,7 +102,7 @@ def agenda_json(request, date=None): res = { "telechat-date": str(data["date"]), - "as-of": str(datetime.datetime.utcnow()), + "as-of": str(datetime.datetime.now(datetime.UTC)), "page-counts": telechat_page_count(date=get_agenda_date(date))._asdict(), "sections": {}, } @@ -547,7 +548,7 @@ def milestones_needing_review(request): ) return render(request, 'iesg/milestones_needing_review.html', - dict(ads=sorted(ad_list, key=lambda ad: ad.plain_name()),)) + dict(ads=sorted(ad_list, key=lambda ad: normalize_for_sorting(ad.plain_name())),)) def photos(request): roles = sorted(Role.objects.filter(group__type='area', group__state='active', name_id='ad'),key=lambda x: "" if x.group.acronym=="gen" else x.group.acronym) @@ -626,3 +627,13 @@ def telechat_agenda_content_view(request, section): content=content.text, content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", ) + +def working_groups(request): + + area_summary, area_totals, ad_summary, noad_summary, ad_totals, noad_totals, totals, wg_summary = get_wg_dashboard_info() + + return render( + request, + "iesg/working_groups.html", + dict(area_summary=area_summary, area_totals=area_totals, ad_summary=ad_summary, noad_summary=noad_summary, ad_totals=ad_totals, noad_totals=noad_totals, totals=totals, wg_summary=wg_summary), + ) diff --git a/ietf/ietfauth/utils.py b/ietf/ietfauth/utils.py index e2893a90f7..0df667fbd2 100644 --- a/ietf/ietfauth/utils.py +++ b/ietf/ietfauth/utils.py @@ -211,9 +211,9 @@ def role_required(*role_names): # specific permissions + def is_authorized_in_doc_stream(user, doc): - """Return whether user is authorized to perform stream duties on - document.""" + """Is user authorized to perform stream duties on doc?""" if has_role(user, ["Secretariat"]): return True @@ -287,7 +287,7 @@ def is_individual_draft_author(user, doc): if not hasattr(user, 'person'): return False - if user.person in doc.authors(): + if user.person in doc.author_persons(): return True return False diff --git a/ietf/ietfauth/views.py b/ietf/ietfauth/views.py index 4219747e12..b5256b14f8 100644 --- a/ietf/ietfauth/views.py +++ b/ietf/ietfauth/views.py @@ -517,7 +517,7 @@ def confirm_password_reset(request, auth): password = data['password'] last_login = None if data['last_login']: - last_login = datetime.datetime.fromtimestamp(data['last_login'], datetime.timezone.utc) + last_login = datetime.datetime.fromtimestamp(data['last_login'], datetime.UTC) except django.core.signing.BadSignature: raise Http404("Invalid or expired auth") diff --git a/ietf/ipr/admin.py b/ietf/ipr/admin.py index afc1952d72..d6a320203b 100644 --- a/ietf/ipr/admin.py +++ b/ietf/ipr/admin.py @@ -1,13 +1,23 @@ -# Copyright The IETF Trust 2010-2020, All Rights Reserved +# Copyright The IETF Trust 2010-2025, All Rights Reserved # -*- coding: utf-8 -*- from django import forms from django.contrib import admin from ietf.name.models import DocRelationshipName -from ietf.ipr.models import (IprDisclosureBase, IprDocRel, IprEvent, - RelatedIpr, HolderIprDisclosure, ThirdPartyIprDisclosure, GenericIprDisclosure, - NonDocSpecificIprDisclosure, LegacyMigrationIprEvent) +from ietf.ipr.models import ( + IprDisclosureBase, + IprDocRel, + IprEvent, + RelatedIpr, + HolderIprDisclosure, + RemovedIprDisclosure, + ThirdPartyIprDisclosure, + GenericIprDisclosure, + NonDocSpecificIprDisclosure, + LegacyMigrationIprEvent, +) +from ietf.utils.admin import SaferTabularInline # ------------------------------------------------------ # ModelAdmins @@ -20,13 +30,13 @@ class Meta: 'sections':forms.TextInput, } -class IprDocRelInline(admin.TabularInline): +class IprDocRelInline(SaferTabularInline): model = IprDocRel form = IprDocRelAdminForm raw_id_fields = ['document'] extra = 1 -class RelatedIprInline(admin.TabularInline): +class RelatedIprInline(SaferTabularInline): model = RelatedIpr raw_id_fields = ['target'] fk_name = 'source' @@ -110,3 +120,9 @@ class LegacyMigrationIprEventAdmin(admin.ModelAdmin): list_filter = ['time', 'type', 'response_due'] raw_id_fields = ['by', 'disclosure', 'message', 'in_reply_to'] admin.site.register(LegacyMigrationIprEvent, LegacyMigrationIprEventAdmin) + +class RemovedIprDisclosureAdmin(admin.ModelAdmin): + pass + + +admin.site.register(RemovedIprDisclosure, RemovedIprDisclosureAdmin) diff --git a/ietf/ipr/mail.py b/ietf/ipr/mail.py index 167b11956c..9bef751b95 100644 --- a/ietf/ipr/mail.py +++ b/ietf/ipr/mail.py @@ -66,9 +66,9 @@ def utc_from_string(s): if date is None: return None elif is_aware(date): - return date.astimezone(datetime.timezone.utc) + return date.astimezone(datetime.UTC) else: - return date.replace(tzinfo=datetime.timezone.utc) + return date.replace(tzinfo=datetime.UTC) # ---------------------------------------------------------------- # Email Functions diff --git a/ietf/ipr/migrations/0005_removediprdisclosure.py b/ietf/ipr/migrations/0005_removediprdisclosure.py new file mode 100644 index 0000000000..400a264579 --- /dev/null +++ b/ietf/ipr/migrations/0005_removediprdisclosure.py @@ -0,0 +1,28 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("ipr", "0004_holderiprdisclosure_is_blanket_disclosure"), + ] + + operations = [ + migrations.CreateModel( + name="RemovedIprDisclosure", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("removed_id", models.PositiveBigIntegerField(unique=True)), + ("reason", models.TextField()), + ], + ), + ] diff --git a/ietf/ipr/migrations/0006_already_removed_ipr.py b/ietf/ipr/migrations/0006_already_removed_ipr.py new file mode 100644 index 0000000000..0e2dbc63eb --- /dev/null +++ b/ietf/ipr/migrations/0006_already_removed_ipr.py @@ -0,0 +1,24 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +from django.db import migrations + + +def forward(apps, schema_editor): + RemovedIprDisclosure = apps.get_model("ipr", "RemovedIprDisclosure") + for id in (6544, 6068): + RemovedIprDisclosure.objects.create( + removed_id=id, + reason="This IPR disclosure was removed as objectively false.", + ) + + +def reverse(apps, schema_editor): + RemovedIprDisclosure = apps.get_model("ipr", "RemovedIprDisclosure") + RemovedIprDisclosure.objects.all().delete() + + +class Migration(migrations.Migration): + dependencies = [ + ("ipr", "0005_removediprdisclosure"), + ] + + operations = [migrations.RunPython(forward, reverse)] diff --git a/ietf/ipr/models.py b/ietf/ipr/models.py index 2d81eb4b42..ea148c2704 100644 --- a/ietf/ipr/models.py +++ b/ietf/ipr/models.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2007-2023, All Rights Reserved +# Copyright The IETF Trust 2007-2025, All Rights Reserved # -*- coding: utf-8 -*- @@ -270,3 +270,7 @@ class LegacyMigrationIprEvent(IprEvent): """A subclass of IprEvent specifically for capturing contents of legacy_url_0, the text of a disclosure submitted by email""" pass + +class RemovedIprDisclosure(models.Model): + removed_id = models.PositiveBigIntegerField(unique=True) + reason = models.TextField() diff --git a/ietf/ipr/resources.py b/ietf/ipr/resources.py index 0d8421cdec..c4d2c436e6 100644 --- a/ietf/ipr/resources.py +++ b/ietf/ipr/resources.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2015-2019, All Rights Reserved +# Copyright The IETF Trust 2015-2025, All Rights Reserved # -*- coding: utf-8 -*- # Autogenerated by the mkresources management command 2015-03-21 14:05 PDT @@ -11,7 +11,7 @@ from ietf import api -from ietf.ipr.models import ( IprDisclosureBase, IprDocRel, HolderIprDisclosure, ThirdPartyIprDisclosure, +from ietf.ipr.models import ( IprDisclosureBase, IprDocRel, HolderIprDisclosure, RemovedIprDisclosure, ThirdPartyIprDisclosure, RelatedIpr, NonDocSpecificIprDisclosure, GenericIprDisclosure, IprEvent, LegacyMigrationIprEvent ) from ietf.person.resources import PersonResource @@ -295,3 +295,18 @@ class Meta: } api.ipr.register(LegacyMigrationIprEventResource()) + + +class RemovedIprDisclosureResource(ModelResource): + class Meta: + queryset = RemovedIprDisclosure.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'removediprdisclosure' + ordering = ['id', ] + filtering = { + "id": ALL, + "removed_id": ALL, + "reason": ALL, + } +api.ipr.register(RemovedIprDisclosureResource()) diff --git a/ietf/ipr/tests.py b/ietf/ipr/tests.py index 74fa540126..53a599e2de 100644 --- a/ietf/ipr/tests.py +++ b/ietf/ipr/tests.py @@ -3,6 +3,7 @@ import datetime +import json from unittest import mock import re @@ -15,6 +16,8 @@ from django.urls import reverse as urlreverse from django.utils import timezone +from django.db.models import Max + import debug # pyflakes:ignore from ietf.api.views import EmailIngestionError @@ -38,13 +41,14 @@ from ietf.ipr.forms import DraftForm, HolderIprDisclosureForm from ietf.ipr.mail import (process_response_email, get_reply_to, get_update_submitter_emails, get_pseudo_submitter, get_holders, get_update_cc_addrs, UndeliverableIprResponseError) -from ietf.ipr.models import (IprDisclosureBase, GenericIprDisclosure, HolderIprDisclosure, +from ietf.ipr.models import (IprDisclosureBase, GenericIprDisclosure, HolderIprDisclosure, RemovedIprDisclosure, ThirdPartyIprDisclosure, IprEvent) from ietf.ipr.templatetags.ipr_filters import no_revisions_message from ietf.ipr.utils import get_genitive, get_ipr_summary, ingest_response_email from ietf.mailtrigger.utils import gather_address_lists from ietf.message.factories import MessageFactory from ietf.message.models import Message +from ietf.person.factories import PersonFactory from ietf.utils.mail import outbox, empty_outbox, get_payload_text from ietf.utils.test_utils import TestCase, login_testing_unauthorized from ietf.utils.text import text_to_dict @@ -125,6 +129,26 @@ def test_showlist(self): self.assertContains(r, "removed as objectively false") ipr.delete() + def test_show_delete(self): + ipr = HolderIprDisclosureFactory() + removed = RemovedIprDisclosure.objects.create( + removed_id=ipr.pk, reason="Removed for reasons" + ) + url = urlreverse("ietf.ipr.views.show", kwargs=dict(id=removed.removed_id)) + r = self.client.get(url) + self.assertContains(r, "Removed for reasons") + q = PyQuery(r.content) + self.assertEqual(len(q("#deletion_warning")), 0) + self.client.login(username="secretary", password="secretary+password") + r = self.client.get(url) + self.assertContains(r, "Removed for reasons") + q = PyQuery(r.content) + self.assertEqual(len(q("#deletion_warning")), 1) + ipr.delete() + r = self.client.get(url) + self.assertContains(r, "Removed for reasons") + q = PyQuery(r.content) + self.assertEqual(len(q("#deletion_warning")), 0) def test_show_posted(self): ipr = HolderIprDisclosureFactory() @@ -1113,3 +1137,56 @@ def test_patent_details_required_unless_blanket(self): val = self.data.pop(pf) self.assertTrue(HolderIprDisclosureForm(data=self.data).is_valid()) self.data[pf] = val + +class JsonSnapshotTests(TestCase): + def test_json_snapshot(self): + h = HolderIprDisclosureFactory() + url = urlreverse("ietf.ipr.views.json_snapshot", kwargs=dict(id=h.id)) + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + dump = json.loads(r.content) + self.assertCountEqual( + [o["model"] for o in dump], + ["ipr.holderiprdisclosure", "ipr.iprdisclosurebase", "person.person"], + ) + h.docs.add(WgRfcFactory()) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + dump = json.loads(r.content) + self.assertCountEqual( + [o["model"] for o in dump], + [ + "ipr.holderiprdisclosure", + "ipr.iprdisclosurebase", + "ipr.iprdocrel", + "person.person", + ], + ) + IprEventFactory( + disclosure=h, + message=MessageFactory(by=PersonFactory()), + in_reply_to=MessageFactory(), + ) + r = self.client.get(url) + self.assertEqual(r.status_code, 200) + dump = json.loads(r.content) + self.assertCountEqual( + [o["model"] for o in dump], + [ + "ipr.holderiprdisclosure", + "ipr.iprdisclosurebase", + "ipr.iprdocrel", + "ipr.iprevent", + "message.message", + "message.message", + "person.person", + "person.person", + "person.person", + "person.person", + ], + ) + no_such_ipr_id = IprDisclosureBase.objects.aggregate(Max("id"))["id__max"] + 1 + url = urlreverse("ietf.ipr.views.json_snapshot", kwargs=dict(id=no_such_ipr_id)) + r = self.client.get(url) + self.assertEqual(r.status_code, 404) diff --git a/ietf/ipr/urls.py b/ietf/ipr/urls.py index 84ed04a66b..2c8a26c624 100644 --- a/ietf/ipr/urls.py +++ b/ietf/ipr/urls.py @@ -21,6 +21,7 @@ url(r'^(?P\d+)/notify/(?Pupdate|posted)/$', views.notify), url(r'^(?P\d+)/post/$', views.post), url(r'^(?P\d+)/state/$', views.state), + url(r'^(?P\d+)/json-snapshot/$', views.json_snapshot), url(r'^update/$', RedirectView.as_view(url=reverse_lazy('ietf.ipr.views.showlist'), permanent=True)), url(r'^update/(?P\d+)/$', views.update), url(r'^new-(?P<_type>(specific|generic|general|third-party))/$', views.new), diff --git a/ietf/ipr/utils.py b/ietf/ipr/utils.py index 7e569a1d1d..bcbb052260 100644 --- a/ietf/ipr/utils.py +++ b/ietf/ipr/utils.py @@ -1,11 +1,16 @@ -# Copyright The IETF Trust 2014-2020, All Rights Reserved +# Copyright The IETF Trust 2014-2025, All Rights Reserved # -*- coding: utf-8 -*- +import json +import debug # pyflakes:ignore + from textwrap import dedent +from django.core import serializers + from ietf.ipr.mail import process_response_email, UndeliverableIprResponseError -import debug # pyflakes:ignore +from ietf.ipr.models import IprDocRel def get_genitive(name): """Return the genitive form of name""" @@ -85,3 +90,18 @@ def ingest_response_email(message: bytes): email_original_message=message, email_attach_traceback=True, ) from err + +def json_dump_disclosure(disclosure): + objs = set() + objs.add(disclosure) + objs.add(disclosure.iprdisclosurebase_ptr) + objs.add(disclosure.by) + objs.update(IprDocRel.objects.filter(disclosure=disclosure)) + objs.update(disclosure.iprevent_set.all()) + objs.update([i.by for i in disclosure.iprevent_set.all()]) + objs.update([i.message for i in disclosure.iprevent_set.all() if i.message ]) + objs.update([i.message.by for i in disclosure.iprevent_set.all() if i.message ]) + objs.update([i.in_reply_to for i in disclosure.iprevent_set.all() if i.in_reply_to ]) + objs.update([i.in_reply_to.by for i in disclosure.iprevent_set.all() if i.in_reply_to ]) + objs = sorted(list(objs),key=lambda o:o.__class__.__name__) + return json.dumps(json.loads(serializers.serialize("json",objs)),indent=4) diff --git a/ietf/ipr/views.py b/ietf/ipr/views.py index 24453df2d2..0a43ff2c27 100644 --- a/ietf/ipr/views.py +++ b/ietf/ipr/views.py @@ -28,11 +28,11 @@ AddCommentForm, AddEmailForm, NotifyForm, StateForm, NonDocSpecificIprDisclosureForm, GenericIprDisclosureForm) from ietf.ipr.models import (IprDisclosureStateName, IprDisclosureBase, - HolderIprDisclosure, GenericIprDisclosure, ThirdPartyIprDisclosure, + HolderIprDisclosure, GenericIprDisclosure, RemovedIprDisclosure, ThirdPartyIprDisclosure, NonDocSpecificIprDisclosure, IprDocRel, RelatedIpr,IprEvent) from ietf.ipr.utils import (get_genitive, get_ipr_summary, - iprs_from_docs, related_docs) + iprs_from_docs, json_dump_disclosure, related_docs) from ietf.mailtrigger.utils import gather_address_lists from ietf.message.models import Message from ietf.message.utils import infer_message @@ -81,7 +81,8 @@ def get_document_emails(ipr): addrs = gather_address_lists('ipr_posted_on_doc',doc=doc).as_strings(compact=False) - author_names = ', '.join(a.person.name for a in doc.documentauthor_set.select_related("person")) + # Get a list of author names for the salutation in the body of the email + author_names = ', '.join(doc.author_names()) context = dict( settings=settings, @@ -152,13 +153,13 @@ def ipr_rfc_number(disclosureDate, thirdPartyDisclosureFlag): # RFC publication date comes from the RFC Editor announcement ipr_rfc_pub_datetime = { - 1310 : datetime.datetime(1992, 3, 13, 0, 0, tzinfo=datetime.timezone.utc), - 1802 : datetime.datetime(1994, 3, 23, 0, 0, tzinfo=datetime.timezone.utc), - 2026 : datetime.datetime(1996, 10, 29, 0, 0, tzinfo=datetime.timezone.utc), - 3668 : datetime.datetime(2004, 2, 18, 0, 0, tzinfo=datetime.timezone.utc), - 3979 : datetime.datetime(2005, 3, 2, 2, 23, tzinfo=datetime.timezone.utc), - 4879 : datetime.datetime(2007, 4, 10, 18, 21, tzinfo=datetime.timezone.utc), - 8179 : datetime.datetime(2017, 5, 31, 23, 1, tzinfo=datetime.timezone.utc), + 1310 : datetime.datetime(1992, 3, 13, 0, 0, tzinfo=datetime.UTC), + 1802 : datetime.datetime(1994, 3, 23, 0, 0, tzinfo=datetime.UTC), + 2026 : datetime.datetime(1996, 10, 29, 0, 0, tzinfo=datetime.UTC), + 3668 : datetime.datetime(2004, 2, 18, 0, 0, tzinfo=datetime.UTC), + 3979 : datetime.datetime(2005, 3, 2, 2, 23, tzinfo=datetime.UTC), + 4879 : datetime.datetime(2007, 4, 10, 18, 21, tzinfo=datetime.UTC), + 8179 : datetime.datetime(2017, 5, 31, 23, 1, tzinfo=datetime.UTC), } if disclosureDate < ipr_rfc_pub_datetime[1310]: @@ -817,7 +818,14 @@ def get_details_tabs(ipr, selected): def show(request, id): """View of individual declaration""" - ipr = get_object_or_404(IprDisclosureBase, id=id).get_child() + ipr = IprDisclosureBase.objects.filter(id=id) + removed = RemovedIprDisclosure.objects.filter(removed_id=id) + if removed.exists(): + return render(request, "ipr/deleted.html", {"removed": removed.get(), "ipr": ipr}) + if not ipr.exists(): + raise Http404 + else: + ipr = ipr.get().get_child() if not has_role(request.user, 'Secretariat'): if ipr.state.slug in ['removed', 'removed_objfalse']: return render(request, "ipr/removed.html", { @@ -901,3 +909,8 @@ def update(request, id): child = ipr.get_child() type = class_to_type[child.__class__.__name__] return new(request, type, updates=id) + +@role_required("Secretariat") +def json_snapshot(request, id): + obj = get_object_or_404(IprDisclosureBase,id=id).get_child() + return HttpResponse(json_dump_disclosure(obj),content_type="application/json") diff --git a/ietf/liaisons/admin.py b/ietf/liaisons/admin.py index 21515ed1a3..d873cce536 100644 --- a/ietf/liaisons/admin.py +++ b/ietf/liaisons/admin.py @@ -7,15 +7,16 @@ from ietf.liaisons.models import ( LiaisonStatement, LiaisonStatementEvent, RelatedLiaisonStatement, LiaisonStatementAttachment ) +from ietf.utils.admin import SaferTabularInline -class RelatedLiaisonStatementInline(admin.TabularInline): +class RelatedLiaisonStatementInline(SaferTabularInline): model = RelatedLiaisonStatement fk_name = 'source' raw_id_fields = ['target'] extra = 1 -class LiaisonStatementAttachmentInline(admin.TabularInline): +class LiaisonStatementAttachmentInline(SaferTabularInline): model = LiaisonStatementAttachment raw_id_fields = ['document'] extra = 1 diff --git a/ietf/liaisons/forms.py b/ietf/liaisons/forms.py index 7483981595..6ceda5ad38 100644 --- a/ietf/liaisons/forms.py +++ b/ietf/liaisons/forms.py @@ -23,7 +23,7 @@ from ietf.liaisons.fields import SearchableLiaisonStatementsField from ietf.liaisons.models import (LiaisonStatement, LiaisonStatementEvent, LiaisonStatementAttachment, LiaisonStatementPurposeName) -from ietf.liaisons.utils import get_person_for_user, is_authorized_individual, OUTGOING_LIAISON_ROLES, \ +from ietf.liaisons.utils import get_person_for_user, OUTGOING_LIAISON_ROLES, \ INCOMING_LIAISON_ROLES from ietf.liaisons.widgets import ButtonWidget, ShowAttachmentsWidget from ietf.name.models import DocRelationshipName @@ -105,7 +105,6 @@ def internal_groups_for_person(person: Optional[Person]): "Secretariat", "IETF Chair", "IAB Chair", - "IAB Executive Director", "Liaison Manager", "Liaison Coordinator", "Authorized Individual", @@ -115,7 +114,7 @@ def internal_groups_for_person(person: Optional[Person]): # Interesting roles, as Group queries queries = [ Q(role__person=person, role__name="chair", acronym="ietf"), - Q(role__person=person, role__name__in=("chair", "execdir"), acronym="iab"), + Q(role__person=person, role__name="chair", acronym="iab"), Q(role__person=person, role__name="ad", type="area", state="active"), Q( role__person=person, @@ -467,25 +466,11 @@ def set_to_fields(self): assert NotImplemented class IncomingLiaisonForm(LiaisonModelForm): - def clean(self): - if 'send' in list(self.data.keys()) and self.get_post_only(): - raise forms.ValidationError('As an IETF Liaison Manager you can not send incoming liaison statements, you only can post them') - return super(IncomingLiaisonForm, self).clean() def is_approved(self): '''Incoming Liaison Statements do not required approval''' return True - def get_post_only(self): - from_groups = self.cleaned_data.get("from_groups") - if ( - has_role(self.user, "Secretariat") - or has_role(self.user, "Liaison Coordinator") - or is_authorized_individual(self.user, from_groups) - ): - return False - return True - def set_from_fields(self): """Configure from "From" fields based on user roles""" qs = external_groups_for_person(self.person) @@ -496,14 +481,18 @@ def set_from_fields(self): self.fields['from_groups'].initial = qs # Note that the IAB chair currently doesn't get to work with incoming liaison statements - if not ( - has_role(self.user, "Secretariat") - or has_role(self.user, "Liaison Coordinator") - ): - self.fields["from_contact"].initial = ( - self.person.role_set.filter(group=qs[0]).first().email.formatted_email() - ) - self.fields["from_contact"].widget.attrs["disabled"] = True + + # Removing this block at the request of the IAB - as a workaround until the new liaison tool is + # create, anyone with access to the form can set any from_contact value + # + # if not ( + # has_role(self.user, "Secretariat") + # or has_role(self.user, "Liaison Coordinator") + # ): + # self.fields["from_contact"].initial = ( + # self.person.role_set.filter(group=qs[0]).first().email.formatted_email() + # ) + # self.fields["from_contact"].widget.attrs["disabled"] = True def set_to_fields(self): '''Set to_groups and to_contacts options and initial value based on user diff --git a/ietf/liaisons/mails.py b/ietf/liaisons/mails.py index 8708c8a078..878aada576 100644 --- a/ietf/liaisons/mails.py +++ b/ietf/liaisons/mails.py @@ -14,7 +14,10 @@ def send_liaison_by_email(request, liaison): subject = 'New Liaison Statement, "%s"' % (liaison.title) from_email = settings.LIAISON_UNIVERSAL_FROM - (to_email, cc) = gather_address_lists('liaison_statement_posted',liaison=liaison) + if liaison.is_outgoing(): + (to_email, cc) = gather_address_lists('liaison_statement_posted_outgoing',liaison=liaison) + else: + (to_email, cc) = gather_address_lists('liaison_statement_posted_incoming',liaison=liaison) bcc = ['statements@ietf.org'] body = render_to_string('liaisons/liaison_mail.txt', dict(liaison=liaison)) diff --git a/ietf/liaisons/tests.py b/ietf/liaisons/tests.py index 1d6cfe0c14..e29045443f 100644 --- a/ietf/liaisons/tests.py +++ b/ietf/liaisons/tests.py @@ -110,65 +110,74 @@ def test_help_pages(self): self.assertEqual(self.client.get('/liaison/help/from_ietf/').status_code, 200) self.assertEqual(self.client.get('/liaison/help/to_ietf/').status_code, 200) + def test_list_other_sdo(self): + GroupFactory(type_id="sdo", state_id="conclude", acronym="third") + GroupFactory(type_id="sdo", state_id="active", acronym="second") + GroupFactory(type_id="sdo", state_id="active", acronym="first") + url = urlreverse("ietf.liaisons.views.list_other_sdo") + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + q = PyQuery(r.content) + self.assertEqual(len(q("h1")), 2) + first_td_elements_text = [e.text for e in q("tr").find("td:first-child a")] + self.assertEqual(first_td_elements_text, ["first", "second", "third"]) class UnitTests(TestCase): - def test_get_cc(self): - from ietf.liaisons.views import get_cc,EMAIL_ALIASES + def test_get_contacts_for_liaison_messages_for_group_primary(self): + from ietf.mailtrigger.utils import get_contacts_for_liaison_messages_for_group_primary,EMAIL_ALIASES # test IETF - cc = get_cc(Group.objects.get(acronym='ietf')) + cc = get_contacts_for_liaison_messages_for_group_primary(Group.objects.get(acronym='ietf')) self.assertTrue(EMAIL_ALIASES['IESG'] in cc) self.assertTrue(EMAIL_ALIASES['IETFCHAIR'] in cc) # test IAB - cc = get_cc(Group.objects.get(acronym='iab')) + cc = get_contacts_for_liaison_messages_for_group_primary(Group.objects.get(acronym='iab')) self.assertTrue(EMAIL_ALIASES['IAB'] in cc) self.assertTrue(EMAIL_ALIASES['IABCHAIR'] in cc) - self.assertTrue(EMAIL_ALIASES['IABEXECUTIVEDIRECTOR'] in cc) # test an Area area = Group.objects.filter(type='area').first() - cc = get_cc(area) + cc = get_contacts_for_liaison_messages_for_group_primary(area) self.assertTrue(EMAIL_ALIASES['IETFCHAIR'] in cc) self.assertTrue(contacts_from_roles([area.ad_role()]) in cc) # test a Working Group wg = Group.objects.filter(type='wg').first() - cc = get_cc(wg) + cc = get_contacts_for_liaison_messages_for_group_primary(wg) self.assertTrue(contacts_from_roles([wg.parent.ad_role()]) in cc) self.assertTrue(contacts_from_roles([wg.get_chair()]) in cc) # test an SDO sdo = RoleFactory(name_id='liaiman',group__type_id='sdo',).group - cc = get_cc(sdo) + cc = get_contacts_for_liaison_messages_for_group_primary(sdo) self.assertTrue(contacts_from_roles([sdo.role_set.filter(name='liaiman').first()]) in cc) # test a cc_contact role cc_contact_role = RoleFactory(name_id='liaison_cc_contact', group=sdo) - cc = get_cc(sdo) + cc = get_contacts_for_liaison_messages_for_group_primary(sdo) self.assertIn(contact_email_from_role(cc_contact_role), cc) - def test_get_contacts_for_group(self): - from ietf.liaisons.views import get_contacts_for_group, EMAIL_ALIASES + def test_get_contacts_for_liaison_messages_for_group_secondary(self): + from ietf.mailtrigger.utils import get_contacts_for_liaison_messages_for_group_secondary,EMAIL_ALIASES - # test explicit + # test explicit group contacts sdo = GroupFactory(type_id='sdo') contact_email = RoleFactory(name_id='liaison_contact', group=sdo).email.address - contacts = get_contacts_for_group(sdo) + contacts = get_contacts_for_liaison_messages_for_group_secondary(sdo) self.assertIsNotNone(contact_email) self.assertIn(contact_email, contacts) # test area area = Group.objects.filter(type='area').first() - contacts = get_contacts_for_group(area) + contacts = get_contacts_for_liaison_messages_for_group_secondary(area) self.assertTrue(area.ad_role().email.address in contacts) # test wg wg = Group.objects.filter(type='wg').first() - contacts = get_contacts_for_group(wg) + contacts = get_contacts_for_liaison_messages_for_group_secondary(wg) self.assertTrue(wg.get_chair().email.address in contacts) # test ietf - contacts = get_contacts_for_group(Group.objects.get(acronym='ietf')) + contacts = get_contacts_for_liaison_messages_for_group_secondary(Group.objects.get(acronym='ietf')) self.assertTrue(EMAIL_ALIASES['IETFCHAIR'] in contacts) # test iab - contacts = get_contacts_for_group(Group.objects.get(acronym='iab')) + contacts = get_contacts_for_liaison_messages_for_group_secondary(Group.objects.get(acronym='iab')) self.assertTrue(EMAIL_ALIASES['IABCHAIR'] in contacts) - self.assertTrue(EMAIL_ALIASES['IABEXECUTIVEDIRECTOR'] in contacts) # test iesg - contacts = get_contacts_for_group(Group.objects.get(acronym='iesg')) + contacts = get_contacts_for_liaison_messages_for_group_secondary(Group.objects.get(acronym='iesg')) self.assertTrue(EMAIL_ALIASES['IESG'] in contacts) def test_needs_approval(self): @@ -205,7 +214,6 @@ def test_ajax(self): self.assertEqual(r.status_code, 200) data = r.json() self.assertEqual(data["error"], False) - self.assertEqual(data["post_only"], False) self.assertTrue('cc' in data) self.assertTrue('needs_approval' in data) self.assertTrue('to_contacts' in data) @@ -365,6 +373,9 @@ def test_approval_process(self): self.assertEqual(len(q('form button[name=approved]')), 0) # check the detail page / authorized + r = self.client.post(url, dict(dead="1")) + self.assertEqual(r.status_code, 403) + mailbox_before = len(outbox) self.client.login(username="ulm-liaiman", password="ulm-liaiman+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) @@ -534,7 +545,6 @@ def test_outgoing_access(self): RoleFactory(name_id='liaison_coordinator', group__acronym='iab', person__user__username='liaison-coordinator') mars = RoleFactory(name_id='chair',person__user__username='marschairman',group__acronym='mars').group RoleFactory(name_id='secr',group=mars,person__user__username='mars-secr') - RoleFactory(name_id='execdir',group=Group.objects.get(acronym='iab'),person__user__username='iab-execdir') url = urlreverse('ietf.liaisons.views.liaison_list') addurl = urlreverse('ietf.liaisons.views.liaison_add', kwargs={'type':'outgoing'}) @@ -592,15 +602,6 @@ def test_outgoing_access(self): r = self.client.get(addurl) self.assertEqual(r.status_code, 200) - # IAB Executive Director - self.assertTrue(self.client.login(username="iab-execdir", password="iab-execdir+password")) - r = self.client.get(url) - self.assertEqual(r.status_code, 200) - q = PyQuery(r.content) - self.assertEqual(len(q("a.btn:contains('New outgoing liaison')")), 1) - r = self.client.get(addurl) - self.assertEqual(r.status_code, 200) - # Liaison Manager has access self.assertTrue(self.client.login(username="ulm-liaiman", password="ulm-liaiman+password")) r = self.client.get(url) @@ -735,7 +736,7 @@ def test_add_incoming_liaison(self): from_groups = [ str(g.pk) for g in Group.objects.filter(type="sdo") ] to_group = Group.objects.get(acronym="mars") submitter = Person.objects.get(user__username="marschairman") - today = date_today(datetime.timezone.utc) + today = date_today(datetime.UTC) related_liaison = liaison r = self.client.post(url, dict(from_groups=from_groups, @@ -795,8 +796,11 @@ def test_add_incoming_liaison(self): self.assertTrue("Liaison Statement" in outbox[-1]["Subject"]) self.assertTrue('to_contacts@' in outbox[-1]['To']) + self.assertTrue(submitter.email_address(), outbox[-1]['To']) self.assertTrue('cc@' in outbox[-1]['Cc']) + + def test_add_outgoing_liaison(self): RoleFactory(name_id='liaiman',group__type_id='sdo', person__user__username='ulm-liaiman') wg = RoleFactory(name_id='chair',person__user__username='marschairman',group__acronym='mars').group @@ -820,7 +824,7 @@ def test_add_outgoing_liaison(self): from_group = Group.objects.get(acronym="mars") to_group = Group.objects.filter(type="sdo")[0] submitter = Person.objects.get(user__username="marschairman") - today = date_today(datetime.timezone.utc) + today = date_today(datetime.UTC) related_liaison = liaison r = self.client.post(url, dict(from_groups=str(from_group.pk), @@ -876,37 +880,7 @@ def test_add_outgoing_liaison(self): self.assertEqual(len(outbox), mailbox_before + 1) self.assertTrue("Liaison Statement" in outbox[-1]["Subject"]) self.assertTrue('aread@' in outbox[-1]['To']) - - def test_add_outgoing_liaison_unapproved_post_only(self): - RoleFactory(name_id='liaiman',group__type_id='sdo', person__user__username='ulm-liaiman') - mars = RoleFactory(name_id='chair',person__user__username='marschairman',group__acronym='mars').group - RoleFactory(name_id='ad',group=mars) - - url = urlreverse('ietf.liaisons.views.liaison_add', kwargs={'type':'outgoing'}) - login_testing_unauthorized(self, "secretary", url) - - # add new - mailbox_before = len(outbox) - from_group = Group.objects.get(acronym="mars") - to_group = Group.objects.filter(type="sdo")[0] - submitter = Person.objects.get(user__username="marschairman") - today = date_today(datetime.timezone.utc) - r = self.client.post(url, - dict(from_groups=str(from_group.pk), - from_contact=submitter.email_address(), - to_groups=str(to_group.pk), - to_contacts='to_contacts@example.com', - approved="", - purpose="info", - title="title", - submitted_date=today.strftime("%Y-%m-%d"), - body="body", - post_only="1", - )) - self.assertEqual(r.status_code, 302) - l = LiaisonStatement.objects.all().order_by("-id")[0] - self.assertEqual(l.state.slug,'pending') - self.assertEqual(len(outbox), mailbox_before + 1) + self.assertTrue(submitter.email_address(), outbox[-1]['Cc']) def test_liaison_add_attachment(self): liaison = LiaisonStatementFactory(deadline=date_today(DEADLINE_TZINFO)+datetime.timedelta(days=1)) @@ -951,26 +925,43 @@ def test_liaison_add_attachment(self): ) def test_liaison_edit_attachment(self): - - attachment = LiaisonStatementAttachmentFactory(document__name='liaiatt-1') - url = urlreverse('ietf.liaisons.views.liaison_edit_attachment', kwargs=dict(object_id=attachment.statement_id,doc_id=attachment.document_id)) + attachment = LiaisonStatementAttachmentFactory(document__name="liaiatt-1") + url = urlreverse( + "ietf.liaisons.views.liaison_edit_attachment", + kwargs=dict( + object_id=attachment.statement_id, doc_id=attachment.document_id + ), + ) login_testing_unauthorized(self, "secretary", url) r = self.client.get(url) self.assertEqual(r.status_code, 200) - post_data = dict(title='New Title') - r = self.client.post(url,post_data) + post_data = dict(title="New Title") + r = self.client.post(url, post_data) attachment = LiaisonStatementAttachment.objects.get(pk=attachment.pk) self.assertEqual(r.status_code, 302) - self.assertEqual(attachment.document.title,'New Title') - - def test_liaison_delete_attachment(self): - attachment = LiaisonStatementAttachmentFactory(document__name='liaiatt-1') - liaison = attachment.statement - url = urlreverse('ietf.liaisons.views.liaison_delete_attachment', kwargs=dict(object_id=liaison.pk,attach_id=attachment.pk)) - login_testing_unauthorized(self, "secretary", url) + self.assertEqual(attachment.document.title, "New Title") + + # ensure attempts to edit attachments not attached to this liaison statement fail + other_attachment = LiaisonStatementAttachmentFactory(document__name="liaiatt-2") + url = urlreverse( + "ietf.liaisons.views.liaison_edit_attachment", + kwargs=dict( + object_id=attachment.statement_id, doc_id=other_attachment.document_id + ), + ) r = self.client.get(url) - self.assertEqual(r.status_code, 302) - self.assertEqual(liaison.liaisonstatementattachment_set.filter(removed=False).count(),0) + self.assertEqual(r.status_code, 404) + r = self.client.post(url, dict(title="New Title")) + self.assertEqual(r.status_code, 404) + + # def test_liaison_delete_attachment(self): + # attachment = LiaisonStatementAttachmentFactory(document__name='liaiatt-1') + # liaison = attachment.statement + # url = urlreverse('ietf.liaisons.views.liaison_delete_attachment', kwargs=dict(object_id=liaison.pk,attach_id=attachment.pk)) + # login_testing_unauthorized(self, "secretary", url) + # r = self.client.get(url) + # self.assertEqual(r.status_code, 302) + # self.assertEqual(liaison.liaisonstatementattachment_set.filter(removed=False).count(),0) def test_in_response(self): '''A statement with purpose=in_response must have related statement specified''' @@ -1074,7 +1065,7 @@ def test_search(self): LiaisonStatementEventFactory(type_id='posted', statement__body="Has recently in its body",statement__from_groups=[GroupFactory(type_id='sdo',acronym='ulm'),]) # Statement 2 s2 = LiaisonStatementEventFactory(type_id='posted', statement__body="That word does not occur here", statement__title="Nor does it occur here") - s2.time=datetime.datetime(2010, 1, 1, tzinfo=datetime.timezone.utc) + s2.time=datetime.datetime(2010, 1, 1, tzinfo=datetime.UTC) s2.save() # test list only, no search filters @@ -1139,17 +1130,6 @@ def test_redirect_for_approval(self): # ------------------------------------------------- # Form validations # ------------------------------------------------- - def test_post_and_send_fail(self): - RoleFactory(name_id='liaiman',person__user__username='ulm-liaiman',group__type_id='sdo',group__acronym='ulm') - GroupFactory(type_id='wg',acronym='mars') - - url = urlreverse('ietf.liaisons.views.liaison_add', kwargs={'type':'incoming'}) - login_testing_unauthorized(self, "ulm-liaiman", url) - - r = self.client.post(url,get_liaison_post_data(),follow=True) - - self.assertEqual(r.status_code, 200) - self.assertContains(r, 'As an IETF Liaison Manager you can not send incoming liaison statements') def test_deadline_field(self): '''Required for action, comment, not info, response''' diff --git a/ietf/liaisons/tests_forms.py b/ietf/liaisons/tests_forms.py index c2afddea65..101c0c8298 100644 --- a/ietf/liaisons/tests_forms.py +++ b/ietf/liaisons/tests_forms.py @@ -94,11 +94,6 @@ def test_all_internal_groups(self): def test_internal_groups_for_person(self): # test relies on the data created in ietf.utils.test_data.make_immutable_test_data() # todo add liaison coordinator when modeled - RoleFactory( - name_id="execdir", - group=Group.objects.get(acronym="iab"), - person__user__username="iab-execdir", - ) RoleFactory( name_id="auth", group__type_id="sdo", @@ -121,7 +116,6 @@ def test_internal_groups_for_person(self): "secretary", "ietf-chair", "iab-chair", - "iab-execdir", "sdo-authperson", ): returned_queryset = internal_groups_for_person( @@ -151,11 +145,6 @@ def test_internal_groups_for_person(self): ) def test_external_groups_for_person(self): - RoleFactory( - name_id="execdir", - group=Group.objects.get(acronym="iab"), - person__user__username="iab-execdir", - ) RoleFactory(name_id="liaison_coordinator", group__acronym="iab", person__user__username="liaison-coordinator") the_sdo = GroupFactory(type_id="sdo", acronym="the-sdo") liaison_manager = RoleFactory(name_id="liaiman", group=the_sdo).person @@ -166,7 +155,6 @@ def test_external_groups_for_person(self): "secretary", "ietf-chair", "iab-chair", - "iab-execdir", "liaison-coordinator", "ad", "sopschairman", diff --git a/ietf/liaisons/urls.py b/ietf/liaisons/urls.py index a4afbfef5d..498df3b965 100644 --- a/ietf/liaisons/urls.py +++ b/ietf/liaisons/urls.py @@ -26,8 +26,8 @@ url(r'^(?P\d+)/$', views.liaison_detail), url(r'^(?P\d+)/addcomment/$', views.add_comment), url(r'^(?P\d+)/edit/$', views.liaison_edit), - url(r'^(?P\d+)/edit-attachment/(?P[A-Za-z0-9._+-]+)$', views.liaison_edit_attachment), - url(r'^(?P\d+)/delete-attachment/(?P[A-Za-z0-9._+-]+)$', views.liaison_delete_attachment), + url(r'^(?P\d+)/edit-attachment/(?P[0-9]+)$', views.liaison_edit_attachment), + url(r'^(?P\d+)/delete-attachment/(?P[0-9]+)$', views.liaison_delete_attachment), url(r'^(?P\d+)/history/$', views.liaison_history), url(r'^(?P\d+)/reply/$', views.liaison_reply), url(r'^(?P\d+)/resend/$', views.liaison_resend), @@ -37,4 +37,5 @@ url(r'^add/$', views.redirect_add), url(r'^for_approval/$', views.redirect_for_approval), url(r'^for_approval/(?P\d+)/$', views.redirect_for_approval), + url(r"^list_other_sdo/$", views.list_other_sdo), ] diff --git a/ietf/liaisons/utils.py b/ietf/liaisons/utils.py index ea06c5988e..469bbc5c87 100644 --- a/ietf/liaisons/utils.py +++ b/ietf/liaisons/utils.py @@ -8,7 +8,6 @@ OUTGOING_LIAISON_ROLES = [ "Area Director", "IAB Chair", - "IAB Executive Director", "IETF Chair", "Liaison Manager", "Liaison Coordinator", diff --git a/ietf/liaisons/views.py b/ietf/liaisons/views.py index 1b7e8d63bb..59c6ea69fc 100644 --- a/ietf/liaisons/views.py +++ b/ietf/liaisons/views.py @@ -7,19 +7,17 @@ from django.contrib import messages from django.urls import reverse as urlreverse -from django.core.exceptions import ValidationError +from django.core.exceptions import ValidationError, ObjectDoesNotExist, PermissionDenied from django.core.validators import validate_email from django.db.models import Q, Prefetch -from django.http import HttpResponse +from django.http import Http404, HttpResponse from django.shortcuts import render, get_object_or_404, redirect import debug # pyflakes:ignore -from ietf.doc.models import Document from ietf.ietfauth.utils import role_required, has_role from ietf.group.models import Group, Role -from ietf.liaisons.models import (LiaisonStatement,LiaisonStatementEvent, - LiaisonStatementAttachment) +from ietf.liaisons.models import LiaisonStatement,LiaisonStatementEvent from ietf.liaisons.utils import (get_person_for_user, can_add_outgoing_liaison, can_add_incoming_liaison, can_edit_liaison,can_submit_liaison_required, can_add_liaison) @@ -29,13 +27,6 @@ from ietf.name.models import LiaisonStatementTagName from ietf.utils.response import permission_denied -EMAIL_ALIASES = { - 'IETFCHAIR':'The IETF Chair ', - 'IESG':'The IESG ', - 'IAB':'The IAB ', - 'IABCHAIR':'The IAB Chair ', - 'IABEXECUTIVEDIRECTOR':'The IAB Executive Director '} - # ------------------------------------------------- # Helper Functions # ------------------------------------------------- @@ -84,8 +75,6 @@ def _find_person_in_emails(liaison, person): return True elif addr in ('iab@iab.org', 'iab-chair@iab.org') and has_role(person.user, "IAB Chair"): return True - elif addr in ('execd@iab.org', ) and has_role(person.user, "IAB Executive Director"): - return True return False @@ -97,66 +86,6 @@ def contacts_from_roles(roles): emails = [ contact_email_from_role(r) for r in roles ] return ','.join(emails) -def get_cc(group): - '''Returns list of emails to use as CC for group. Simplified refactor of IETFHierarchy - get_cc() and get_from_cc() - ''' - emails = [] - - # role based CCs - if group.acronym in ('ietf','iesg'): - emails.append(EMAIL_ALIASES['IESG']) - emails.append(EMAIL_ALIASES['IETFCHAIR']) - elif group.acronym in ('iab'): - emails.append(EMAIL_ALIASES['IAB']) - emails.append(EMAIL_ALIASES['IABCHAIR']) - emails.append(EMAIL_ALIASES['IABEXECUTIVEDIRECTOR']) - elif group.type_id == 'area': - emails.append(EMAIL_ALIASES['IETFCHAIR']) - ad_roles = group.role_set.filter(name='ad') - emails.extend([ contact_email_from_role(r) for r in ad_roles ]) - elif group.type_id == 'wg': - ad_roles = group.parent.role_set.filter(name='ad') - emails.extend([ contact_email_from_role(r) for r in ad_roles ]) - chair_roles = group.role_set.filter(name='chair') - emails.extend([ contact_email_from_role(r) for r in chair_roles ]) - if group.list_email: - emails.append('{} Discussion List <{}>'.format(group.name,group.list_email)) - elif group.type_id == 'sdo': - liaiman_roles = group.role_set.filter(name='liaiman') - emails.extend([ contact_email_from_role(r) for r in liaiman_roles ]) - - # explicit CCs - liaison_cc_roles = group.role_set.filter(name='liaison_cc_contact') - emails.extend([ contact_email_from_role(r) for r in liaison_cc_roles ]) - - return emails - -def get_contacts_for_group(group): - '''Returns default contacts for groups as a comma separated string''' - # use explicit default contacts if defined - explicit_contacts = contacts_from_roles(group.role_set.filter(name='liaison_contact')) - if explicit_contacts: - return explicit_contacts - - # otherwise construct based on group type - contacts = [] - if group.type_id == 'area': - roles = group.role_set.filter(name='ad') - contacts.append(contacts_from_roles(roles)) - elif group.type_id == 'wg': - roles = group.role_set.filter(name='chair') - contacts.append(contacts_from_roles(roles)) - elif group.acronym == 'ietf': - contacts.append(EMAIL_ALIASES['IETFCHAIR']) - elif group.acronym == 'iab': - contacts.append(EMAIL_ALIASES['IABCHAIR']) - contacts.append(EMAIL_ALIASES['IABEXECUTIVEDIRECTOR']) - elif group.acronym == 'iesg': - contacts.append(EMAIL_ALIASES['IESG']) - - return ','.join(contacts) - def get_details_tabs(stmt, selected): return [ t + (t[0].lower() == selected.lower(),) @@ -171,7 +100,7 @@ def needs_approval(group,person): user = person.user if group.acronym in ('ietf','iesg') and has_role(user, 'IETF Chair'): return False - if group.acronym == 'iab' and (has_role(user,'IAB Chair') or has_role(user,'IAB Executive Director')): + if group.acronym == 'iab' and has_role(user,'IAB Chair'): return False if group.type_id == 'area' and group.role_set.filter(name='ad',person=person): return False @@ -189,29 +118,14 @@ def normalize_sort(request): return sort, order_by -def post_only(group,person): - '''Returns true if the user is restricted to post_only (vs. post_and_send) for this - group. This is for incoming liaison statements. - - Secretariat have full access. - - Authorized Individuals have full access for the group they are associated with - - Liaison Managers can post only - ''' - if group.type_id == "sdo" and ( - not ( - has_role(person.user, "Secretariat") - or has_role(person.user, "Liaison Coordinator") - or group.role_set.filter(name="auth", person=person) - ) - ): - return True - else: - return False # ------------------------------------------------- # Ajax Functions # ------------------------------------------------- @can_submit_liaison_required def ajax_get_liaison_info(request): + from ietf.mailtrigger.utils import get_contacts_for_liaison_messages_for_group_primary,get_contacts_for_liaison_messages_for_group_secondary + '''Returns dictionary of info to update entry form given the groups that have been selected ''' @@ -228,20 +142,18 @@ def ajax_get_liaison_info(request): cc = [] does_need_approval = [] - can_post_only = [] to_contacts = [] response_contacts = [] - result = {'response_contacts':[],'to_contacts': [], 'cc': [], 'needs_approval': False, 'post_only': False, 'full_list': []} + result = {'response_contacts':[],'to_contacts': [], 'cc': [], 'needs_approval': False, 'full_list': []} for group in from_groups: - cc.extend(get_cc(group)) + cc.extend(get_contacts_for_liaison_messages_for_group_primary(group)) does_need_approval.append(needs_approval(group,person)) - can_post_only.append(post_only(group,person)) - response_contacts.append(get_contacts_for_group(group)) + response_contacts.append(get_contacts_for_liaison_messages_for_group_secondary(group)) for group in to_groups: - cc.extend(get_cc(group)) - to_contacts.append(get_contacts_for_group(group)) + cc.extend(get_contacts_for_liaison_messages_for_group_primary(group)) + to_contacts.append(get_contacts_for_liaison_messages_for_group_secondary(group)) # if there are from_groups and any need approval if does_need_approval: @@ -252,12 +164,15 @@ def ajax_get_liaison_info(request): else: does_need_approval = True - result.update({'error': False, - 'cc': list(set(cc)), - 'response_contacts':list(set(response_contacts)), - 'to_contacts': list(set(to_contacts)), - 'needs_approval': does_need_approval, - 'post_only': any(can_post_only)}) + result.update( + { + "error": False, + "cc": list(set(cc)), + "response_contacts": list(set(response_contacts)), + "to_contacts": list(set(to_contacts)), + "needs_approval": does_need_approval, + } + ) json_result = json.dumps(result) return HttpResponse(json_result, content_type='application/json') @@ -381,23 +296,29 @@ def liaison_history(request, object_id): def liaison_delete_attachment(request, object_id, attach_id): liaison = get_object_or_404(LiaisonStatement, pk=object_id) - attach = get_object_or_404(LiaisonStatementAttachment, pk=attach_id) + if not can_edit_liaison(request.user, liaison): permission_denied(request, "You are not authorized for this action.") - - # FIXME: this view should use POST instead of GET when deleting - attach.removed = True - attach.save() - - # create event - LiaisonStatementEvent.objects.create( - type_id='modified', - by=get_person_for_user(request.user), - statement=liaison, - desc='Attachment Removed: {}'.format(attach.document.title) - ) - messages.success(request, 'Attachment Deleted') - return redirect('ietf.liaisons.views.liaison_detail', object_id=liaison.pk) + else: + permission_denied(request, "This operation is temporarily unavailable. Ask the secretariat to mark the attachment as removed using the admin.") + + # The following will be replaced with a different approach in the next generation of the liaison tool + # attach = get_object_or_404(LiaisonStatementAttachment, pk=attach_id) + + # # FIXME: this view should use POST instead of GET when deleting + # attach.removed = True + # debug.say("Got here") + # attach.save() + + # # create event + # LiaisonStatementEvent.objects.create( + # type_id='modified', + # by=get_person_for_user(request.user), + # statement=liaison, + # desc='Attachment Removed: {}'.format(attach.document.title) + # ) + # messages.success(request, 'Attachment Deleted') + # return redirect('ietf.liaisons.views.liaison_detail', object_id=liaison.pk) def liaison_detail(request, object_id): liaison = get_object_or_404(LiaisonStatement, pk=object_id) @@ -408,22 +329,28 @@ def liaison_detail(request, object_id): if request.method == 'POST': - if request.POST.get('approved'): - liaison.change_state(state_id='approved',person=person) - liaison.change_state(state_id='posted',person=person) - send_liaison_by_email(request, liaison) - messages.success(request,'Liaison Statement Approved and Posted') - elif request.POST.get('dead'): - liaison.change_state(state_id='dead',person=person) - messages.success(request,'Liaison Statement Killed') - elif request.POST.get('resurrect'): - liaison.change_state(state_id='pending',person=person) - messages.success(request,'Liaison Statement Resurrected') - elif request.POST.get('do_action_taken') and can_take_care: + if request.POST.get('do_action_taken') and can_take_care: liaison.tags.remove('required') liaison.tags.add('taken') can_take_care = False messages.success(request,'Action handled') + else: + if can_edit: + if request.POST.get('approved'): + liaison.change_state(state_id='approved',person=person) + liaison.change_state(state_id='posted',person=person) + send_liaison_by_email(request, liaison) + messages.success(request,'Liaison Statement Approved and Posted') + elif request.POST.get('dead'): + liaison.change_state(state_id='dead',person=person) + messages.success(request,'Liaison Statement Killed') + elif request.POST.get('resurrect'): + liaison.change_state(state_id='pending',person=person) + messages.success(request,'Liaison Statement Resurrected') + else: + pass + else: + raise PermissionDenied() relations_by = [i.target for i in liaison.source_of_set.filter(target__state__slug='posted')] relations_to = [i.source for i in liaison.target_of_set.filter(source__state__slug='posted')] @@ -447,7 +374,11 @@ def liaison_edit(request, object_id): def liaison_edit_attachment(request, object_id, doc_id): '''Edit the Liaison Statement attachment title''' liaison = get_object_or_404(LiaisonStatement, pk=object_id) - doc = get_object_or_404(Document, pk=doc_id) + try: + doc = liaison.attachments.get(pk=doc_id) + except ObjectDoesNotExist: + raise Http404 + if not can_edit_liaison(request.user, liaison): permission_denied(request, "You are not authorized for this action.") @@ -578,3 +509,17 @@ def liaison_resend(request, object_id): messages.success(request,'Liaison Statement resent') return redirect('ietf.liaisons.views.liaison_list') + +@role_required("Secretariat", "IAB", "Liaison Coordinator", "Liaison Manager") +def list_other_sdo(request): + def _sdo_order_key(obj:Group)-> tuple[str,str]: + state_order = { + "active" : "a", + "conclude": "b", + } + return (state_order.get(obj.state.slug,f"c{obj.state.slug}"), obj.acronym) + + sdos = sorted(list(Group.objects.filter(type="sdo")),key = _sdo_order_key) + for sdo in sdos: + sdo.liaison_managers =[r.person for r in sdo.role_set.filter(name="liaiman")] + return render(request,"liaisons/list_other_sdo.html",dict(sdos=sdos)) diff --git a/ietf/liaisons/widgets.py b/ietf/liaisons/widgets.py index 74368e83f2..48db8af0a3 100644 --- a/ietf/liaisons/widgets.py +++ b/ietf/liaisons/widgets.py @@ -26,7 +26,9 @@ def render(self, name, value, **kwargs): html += '%s' % conditional_escape(i) required_str = 'Please fill in %s to attach a new file' % conditional_escape(self.required_label) html += '%s' % conditional_escape(required_str) - html += '' % conditional_escape(self.label) + html += ''.format( + f"id_{name}", conditional_escape(self.label) + ) return mark_safe(html) diff --git a/ietf/mailtrigger/admin.py b/ietf/mailtrigger/admin.py index a60fd5b072..8c73f2ae02 100644 --- a/ietf/mailtrigger/admin.py +++ b/ietf/mailtrigger/admin.py @@ -1,9 +1,10 @@ -# Copyright The IETF Trust 2015-2019, All Rights Reserved +# Copyright The IETF Trust 2015-2025, All Rights Reserved from django.contrib import admin +from simple_history.admin import SimpleHistoryAdmin from ietf.mailtrigger.models import MailTrigger, Recipient -class RecipientAdmin(admin.ModelAdmin): +class RecipientAdmin(SimpleHistoryAdmin): list_display = [ 'slug', 'desc', 'template', 'has_code', ] def has_code(self, obj): return hasattr(obj,'gather_%s'%obj.slug) @@ -11,7 +12,7 @@ def has_code(self, obj): admin.site.register(Recipient, RecipientAdmin) -class MailTriggerAdmin(admin.ModelAdmin): +class MailTriggerAdmin(SimpleHistoryAdmin): list_display = [ 'slug', 'desc', ] filter_horizontal = [ 'to', 'cc', ] admin.site.register(MailTrigger, MailTriggerAdmin) diff --git a/ietf/mailtrigger/forms.py b/ietf/mailtrigger/forms.py index 366c429d8c..8d13c5edf3 100644 --- a/ietf/mailtrigger/forms.py +++ b/ietf/mailtrigger/forms.py @@ -11,6 +11,7 @@ class CcSelectForm(forms.Form): expansions = dict() # type: Dict[str, List[str]] cc_choices = forms.MultipleChoiceField( + required=False, label='Cc', choices=[], widget=forms.CheckboxSelectMultiple(), diff --git a/ietf/mailtrigger/migrations/0007_historicalrecipient_historicalmailtrigger.py b/ietf/mailtrigger/migrations/0007_historicalrecipient_historicalmailtrigger.py new file mode 100644 index 0000000000..d23b72d737 --- /dev/null +++ b/ietf/mailtrigger/migrations/0007_historicalrecipient_historicalmailtrigger.py @@ -0,0 +1,122 @@ +# Copyright The IETF Trust 2025, All Rights Reserved +from io import StringIO + +from django.conf import settings +from django.core import management +from django.db import migrations, models +import django.db.models.deletion +import simple_history.models + +from ietf.utils.log import log + + +def forward(apps, schema_editor): + # Fill in history for existing data using the populate_history management command + captured_stdout = StringIO() + captured_stderr = StringIO() + try: + management.call_command( + "populate_history", + "mailtrigger.MailTrigger", + "mailtrigger.Recipient", + stdout=captured_stdout, + stderr=captured_stderr, + ) + except management.CommandError as err: + log( + "Failed to populate history for mailtrigger models.\n" + "\n" + f"stdout:\n{captured_stdout.getvalue() or ''}\n" + "\n" + f"stderr:\n{captured_stderr.getvalue() or ''}\n" + ) + raise RuntimeError("Failed to populate history for mailtrigger models") from err + log( + "Populated history for mailtrigger models.\n" + "\n" + f"stdout:\n{captured_stdout.getvalue() or ''}\n" + "\n" + f"stderr:\n{captured_stderr.getvalue() or ''}\n" + ) + + +def reverse(apps, schema_editor): + pass # nothing to do + + +class Migration(migrations.Migration): + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("mailtrigger", "0006_call_for_adoption_and_last_call_issued"), + ] + + operations = [ + migrations.CreateModel( + name="HistoricalRecipient", + fields=[ + ("slug", models.CharField(db_index=True, max_length=32)), + ("desc", models.TextField(blank=True)), + ("template", models.TextField(blank=True, null=True)), + ("history_id", models.AutoField(primary_key=True, serialize=False)), + ("history_date", models.DateTimeField(db_index=True)), + ("history_change_reason", models.CharField(max_length=100, null=True)), + ( + "history_type", + models.CharField( + choices=[("+", "Created"), ("~", "Changed"), ("-", "Deleted")], + max_length=1, + ), + ), + ( + "history_user", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "verbose_name": "historical recipient", + "verbose_name_plural": "historical recipients", + "ordering": ("-history_date", "-history_id"), + "get_latest_by": ("history_date", "history_id"), + }, + bases=(simple_history.models.HistoricalChanges, models.Model), + ), + migrations.CreateModel( + name="HistoricalMailTrigger", + fields=[ + ("slug", models.CharField(db_index=True, max_length=64)), + ("desc", models.TextField(blank=True)), + ("history_id", models.AutoField(primary_key=True, serialize=False)), + ("history_date", models.DateTimeField(db_index=True)), + ("history_change_reason", models.CharField(max_length=100, null=True)), + ( + "history_type", + models.CharField( + choices=[("+", "Created"), ("~", "Changed"), ("-", "Deleted")], + max_length=1, + ), + ), + ( + "history_user", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="+", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "verbose_name": "historical mail trigger", + "verbose_name_plural": "historical mail triggers", + "ordering": ("-history_date", "-history_id"), + "get_latest_by": ("history_date", "history_id"), + }, + bases=(simple_history.models.HistoricalChanges, models.Model), + ), + migrations.RunPython(forward, reverse), + ] diff --git a/ietf/mailtrigger/migrations/0008_liaison_statement_incoming_and_outgoing_posted.py b/ietf/mailtrigger/migrations/0008_liaison_statement_incoming_and_outgoing_posted.py new file mode 100644 index 0000000000..189a783a2e --- /dev/null +++ b/ietf/mailtrigger/migrations/0008_liaison_statement_incoming_and_outgoing_posted.py @@ -0,0 +1,72 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.db import migrations + + +def forward(apps, schema_editor): + Mailtrigger = apps.get_model("mailtrigger", "MailTrigger") + Recipient = apps.get_model("mailtrigger", "Recipient") + recipients_to = Recipient.objects.get(pk="liaison_to_contacts") + recipients_cc = list( + Recipient.objects.filter( + slug__in=( + "liaison_cc", + "liaison_coordinators", + "liaison_response_contacts", + "liaison_technical_contacts", + ) + ) + ) + recipient_from = Recipient.objects.get(pk="liaison_from_contact") + + liaison_posted_outgoing = Mailtrigger.objects.create( + slug="liaison_statement_posted_outgoing", + desc="Recipients for a message when a new outgoing liaison statement is posted", + ) + liaison_posted_outgoing.to.add(recipients_to) + liaison_posted_outgoing.cc.add(*recipients_cc) + liaison_posted_outgoing.cc.add(recipient_from) + + liaison_posted_incoming = Mailtrigger.objects.create( + slug="liaison_statement_posted_incoming", + desc="Recipients for a message when a new incoming liaison statement is posted", + ) + liaison_posted_incoming.to.add(recipients_to) + liaison_posted_incoming.cc.add(*recipients_cc) + + Mailtrigger.objects.filter(slug=("liaison_statement_posted")).delete() + + +def reverse(apps, schema_editor): + Mailtrigger = apps.get_model("mailtrigger", "MailTrigger") + Recipient = apps.get_model("mailtrigger", "Recipient") + + Mailtrigger.objects.filter( + slug__in=( + "liaison_statement_posted_outgoing", + "liaison_statement_posted_incoming", + ) + ).delete() + + liaison_statement_posted = Mailtrigger.objects.create( + slug="liaison_statement_posted", + desc="Recipients for a message when a new liaison statement is posted", + ) + + liaison_to_contacts = Recipient.objects.get(slug="liaison_to_contacts") + recipients_ccs = Recipient.objects.filter( + slug__in=( + "liaison_cc", + "liaison_coordinators", + "liaison_response_contacts", + "liaison_technical_contacts", + ) + ) + liaison_statement_posted.to.add(liaison_to_contacts) + liaison_statement_posted.cc.add(*recipients_ccs) + + +class Migration(migrations.Migration): + dependencies = [("mailtrigger", "0007_historicalrecipient_historicalmailtrigger")] + + operations = [migrations.RunPython(forward, reverse)] diff --git a/ietf/mailtrigger/models.py b/ietf/mailtrigger/models.py index 66b7139fa5..435729f893 100644 --- a/ietf/mailtrigger/models.py +++ b/ietf/mailtrigger/models.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2015-2020, All Rights Reserved +# Copyright The IETF Trust 2015-2025, All Rights Reserved # -*- coding: utf-8 -*- @@ -7,6 +7,8 @@ from email.utils import parseaddr +from simple_history.models import HistoricalRecords + from ietf.doc.utils_bofreq import bofreq_editors, bofreq_responsible from ietf.utils.mail import formataddr, get_email_addresses_from_text from ietf.group.models import Group, Role @@ -38,6 +40,7 @@ class MailTrigger(models.Model): desc = models.TextField(blank=True) to = models.ManyToManyField('mailtrigger.Recipient', blank=True, related_name='used_in_to') cc = models.ManyToManyField('mailtrigger.Recipient', blank=True, related_name='used_in_cc') + history = HistoricalRecords() class Meta: ordering = ["slug"] @@ -49,6 +52,7 @@ class Recipient(models.Model): slug = models.CharField(max_length=32, primary_key=True) desc = models.TextField(blank=True) template = models.TextField(null=True, blank=True) + history = HistoricalRecords() class Meta: ordering = ["slug"] diff --git a/ietf/mailtrigger/resources.py b/ietf/mailtrigger/resources.py index eb5466618a..daca055bf4 100644 --- a/ietf/mailtrigger/resources.py +++ b/ietf/mailtrigger/resources.py @@ -7,7 +7,7 @@ from ietf import api -from ietf.mailtrigger.models import Recipient, MailTrigger +from ietf.mailtrigger.models import MailTrigger, Recipient class RecipientResource(ModelResource): @@ -37,3 +37,43 @@ class Meta: } api.mailtrigger.register(MailTriggerResource()) +from ietf.utils.resources import UserResource +class HistoricalMailTriggerResource(ModelResource): + history_user = ToOneField(UserResource, 'history_user', null=True) + class Meta: + queryset = MailTrigger.history.model.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'historicalmailtrigger' + ordering = ['history_id', ] + filtering = { + "slug": ALL, + "desc": ALL, + "history_id": ALL, + "history_date": ALL, + "history_change_reason": ALL, + "history_type": ALL, + "history_user": ALL_WITH_RELATIONS, + } +api.mailtrigger.register(HistoricalMailTriggerResource()) + +from ietf.utils.resources import UserResource +class HistoricalRecipientResource(ModelResource): + history_user = ToOneField(UserResource, 'history_user', null=True) + class Meta: + queryset = Recipient.history.model.objects.all() + serializer = api.Serializer() + cache = SimpleCache() + #resource_name = 'historicalrecipient' + ordering = ['history_id', ] + filtering = { + "slug": ALL, + "desc": ALL, + "template": ALL, + "history_id": ALL, + "history_date": ALL, + "history_change_reason": ALL, + "history_type": ALL, + "history_user": ALL_WITH_RELATIONS, + } +api.mailtrigger.register(HistoricalRecipientResource()) diff --git a/ietf/mailtrigger/utils.py b/ietf/mailtrigger/utils.py index 9915eae3fd..bcdaf5e44e 100644 --- a/ietf/mailtrigger/utils.py +++ b/ietf/mailtrigger/utils.py @@ -9,6 +9,14 @@ from ietf.utils.mail import excludeaddrs +EMAIL_ALIASES = { + "IETFCHAIR": "The IETF Chair ", + "IESG": "The IESG ", + "IAB": "The IAB ", + "IABCHAIR": "The IAB Chair ", +} + + class AddrLists(namedtuple("AddrLists", ["to", "cc"])): __slots__ = () @@ -66,6 +74,69 @@ def get_mailtrigger(slug, create_from_slug_if_not_exists, desc_if_not_exists): return mailtrigger +def get_contacts_for_liaison_messages_for_group_primary(group): + from ietf.liaisons.views import contact_email_from_role + + '''Returns list of emails to use in liaison message for group + ''' + emails = [] + + # role based emails + if group.acronym in ('ietf','iesg'): + emails.append(EMAIL_ALIASES['IESG']) + emails.append(EMAIL_ALIASES['IETFCHAIR']) + elif group.acronym in ('iab'): + emails.append(EMAIL_ALIASES['IAB']) + emails.append(EMAIL_ALIASES['IABCHAIR']) + elif group.type_id == 'area': + emails.append(EMAIL_ALIASES['IETFCHAIR']) + ad_roles = group.role_set.filter(name='ad') + emails.extend([ contact_email_from_role(r) for r in ad_roles ]) + elif group.type_id == 'wg': + ad_roles = group.parent.role_set.filter(name='ad') + emails.extend([ contact_email_from_role(r) for r in ad_roles ]) + chair_roles = group.role_set.filter(name='chair') + emails.extend([ contact_email_from_role(r) for r in chair_roles ]) + if group.list_email: + emails.append('{} Discussion List <{}>'.format(group.name,group.list_email)) + elif group.type_id == 'sdo': + liaiman_roles = group.role_set.filter(name='liaiman') + emails.extend([ contact_email_from_role(r) for r in liaiman_roles ]) + + # explicit CCs + liaison_cc_roles = group.role_set.filter(name='liaison_cc_contact') + emails.extend([ contact_email_from_role(r) for r in liaison_cc_roles ]) + + return emails + + +def get_contacts_for_liaison_messages_for_group_secondary(group): + from ietf.liaisons.views import contacts_from_roles + + '''Returns default contacts for groups as a comma separated string''' + # use explicit default contacts if defined + explicit_contacts = contacts_from_roles(group.role_set.filter(name='liaison_contact')) + if explicit_contacts: + return explicit_contacts + + # otherwise construct based on group type + contacts = [] + if group.type_id == 'area': + roles = group.role_set.filter(name='ad') + contacts.append(contacts_from_roles(roles)) + elif group.type_id == 'wg': + roles = group.role_set.filter(name='chair') + contacts.append(contacts_from_roles(roles)) + elif group.acronym == 'ietf': + contacts.append(EMAIL_ALIASES['IETFCHAIR']) + elif group.acronym == 'iab': + contacts.append(EMAIL_ALIASES['IABCHAIR']) + elif group.acronym == 'iesg': + contacts.append(EMAIL_ALIASES['IESG']) + + return ','.join(contacts) + + def gather_relevant_expansions(**kwargs): def starts_with(prefix): return MailTrigger.objects.filter(slug__startswith=prefix).values_list( diff --git a/ietf/meeting/admin.py b/ietf/meeting/admin.py index d886a9a4b6..03abf5c029 100644 --- a/ietf/meeting/admin.py +++ b/ietf/meeting/admin.py @@ -10,6 +10,7 @@ SessionPresentation, ImportantDate, SlideSubmission, SchedulingEvent, BusinessConstraint, ProceedingsMaterial, MeetingHost, Registration, RegistrationTicket, AttendanceTypeName) +from ietf.utils.admin import SaferTabularInline class UrlResourceAdmin(admin.ModelAdmin): @@ -18,7 +19,7 @@ class UrlResourceAdmin(admin.ModelAdmin): raw_id_fields = ['room', ] admin.site.register(UrlResource, UrlResourceAdmin) -class UrlResourceInline(admin.TabularInline): +class UrlResourceInline(SaferTabularInline): model = UrlResource class RoomAdmin(admin.ModelAdmin): @@ -28,7 +29,7 @@ class RoomAdmin(admin.ModelAdmin): admin.site.register(Room, RoomAdmin) -class RoomInline(admin.TabularInline): +class RoomInline(SaferTabularInline): model = Room class MeetingAdmin(admin.ModelAdmin): @@ -93,7 +94,7 @@ def name_lower(self, instance): admin.site.register(Constraint, ConstraintAdmin) -class SchedulingEventInline(admin.TabularInline): +class SchedulingEventInline(SaferTabularInline): model = SchedulingEvent raw_id_fields = ["by"] @@ -244,7 +245,7 @@ def queryset(self, request, queryset): return queryset.filter(tickets__attendance_type__slug=self.value()).distinct() return queryset -class RegistrationTicketInline(admin.TabularInline): +class RegistrationTicketInline(SaferTabularInline): model = RegistrationTicket class RegistrationAdmin(admin.ModelAdmin): diff --git a/ietf/meeting/forms.py b/ietf/meeting/forms.py index b6b1a1591f..e5b1697f86 100644 --- a/ietf/meeting/forms.py +++ b/ietf/meeting/forms.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2016-2023, All Rights Reserved +# Copyright The IETF Trust 2016-2025, All Rights Reserved # -*- coding: utf-8 -*- @@ -15,19 +15,24 @@ from django.core import validators from django.core.exceptions import ValidationError from django.forms import BaseInlineFormSet +from django.template.defaultfilters import pluralize from django.utils.functional import cached_property +from django.utils.safestring import mark_safe import debug # pyflakes:ignore from ietf.doc.models import Document, State, NewRevisionDocEvent from ietf.group.models import Group from ietf.group.utils import groups_managed_by -from ietf.meeting.models import Session, Meeting, Schedule, COUNTRIES, TIMEZONES, TimeSlot, Room +from ietf.meeting.models import (Session, Meeting, Schedule, COUNTRIES, TIMEZONES, TimeSlot, Room, + Constraint, ResourceAssociation) from ietf.meeting.helpers import get_next_interim_number, make_materials_directories from ietf.meeting.helpers import is_interim_meeting_approved, get_next_agenda_name from ietf.message.models import Message -from ietf.name.models import TimeSlotTypeName, SessionPurposeName +from ietf.name.models import TimeSlotTypeName, SessionPurposeName, TimerangeName, ConstraintName +from ietf.person.fields import SearchablePersonsField from ietf.person.models import Person +from ietf.utils import log from ietf.utils.fields import ( DatepickerDateField, DatepickerSplitDateTimeWidget, @@ -35,9 +40,14 @@ ModelMultipleChoiceField, MultiEmailField, ) +from ietf.utils.html import clean_text_field from ietf.utils.validators import ( validate_file_size, validate_mime_type, validate_file_extension, validate_no_html_frame) +NUM_SESSION_CHOICES = (('', '--Please select'), ('1', '1'), ('2', '2')) +SESSION_TIME_RELATION_CHOICES = (('', 'No preference'),) + Constraint.TIME_RELATION_CHOICES +JOINT_FOR_SESSION_CHOICES = (('1', 'First session'), ('2', 'Second session'), ('3', 'Third session'), ) + # ------------------------------------------------- # Helpers # ------------------------------------------------- @@ -74,6 +84,27 @@ def duration_string(duration): return string +def allowed_conflicting_groups(): + return Group.objects.filter( + type__in=['wg', 'ag', 'rg', 'rag', 'program', 'edwg'], + state__in=['bof', 'proposed', 'active']) + + +def check_conflict(groups, source_group): + ''' + Takes a string which is a list of group acronyms. Checks that they are all active groups + ''' + # convert to python list (allow space or comma separated lists) + items = groups.replace(',', ' ').split() + active_groups = allowed_conflicting_groups() + for group in items: + if group == source_group.acronym: + raise forms.ValidationError("Cannot declare a conflict with the same group: %s" % group) + + if not active_groups.filter(acronym=group): + raise forms.ValidationError("Invalid or inactive group acronym: %s" % group) + + # ------------------------------------------------- # Forms # ------------------------------------------------- @@ -753,6 +784,9 @@ def __init__(self, group, *args, **kwargs): self.fields['purpose'].queryset = SessionPurposeName.objects.filter(pk__in=session_purposes) if not group.features.acts_like_wg: self.fields['requested_duration'].durations = [datetime.timedelta(minutes=m) for m in range(30, 241, 30)] + # add bootstrap classes + self.fields['purpose'].widget.attrs.update({'class': 'form-select'}) + self.fields['type'].widget.attrs.update({'class': 'form-select', 'aria-label': 'session type'}) class Meta: model = Session @@ -837,3 +871,296 @@ def sessiondetailsformset_factory(min_num=1, max_num=3): max_num=max_num, extra=max_num, # only creates up to max_num total ) + + +class SessionRequestStatusForm(forms.Form): + message = forms.CharField(widget=forms.Textarea(attrs={'rows': '3', 'cols': '80'}), strip=False) + + +class NameModelMultipleChoiceField(ModelMultipleChoiceField): + def label_from_instance(self, name): + return name.desc + + +class SessionRequestForm(forms.Form): + num_session = forms.ChoiceField( + choices=NUM_SESSION_CHOICES, + label="Number of sessions") + # session fields are added in __init__() + session_time_relation = forms.ChoiceField( + choices=SESSION_TIME_RELATION_CHOICES, + required=False, + label="Time between two sessions") + attendees = forms.IntegerField(label="Number of Attendees") + # FIXME: it would cleaner to have these be + # ModelMultipleChoiceField, and just customize the widgetry, that + # way validation comes for free (applies to this CharField and the + # constraints dynamically instantiated in __init__()) + joint_with_groups = forms.CharField(max_length=255, required=False) + joint_with_groups_selector = forms.ChoiceField(choices=[], required=False) # group select widget for prev field + joint_for_session = forms.ChoiceField(choices=JOINT_FOR_SESSION_CHOICES, required=False) + comments = forms.CharField( + max_length=200, + label='Special Requests', + help_text='i.e. restrictions on meeting times / days, etc. (limit 200 characters)', + required=False) + third_session = forms.BooleanField( + required=False, + help_text="Help") + resources = forms.MultipleChoiceField( + widget=forms.CheckboxSelectMultiple, + required=False, + label='Resources Requested') + bethere = SearchablePersonsField( + label="Participants who must be present", + required=False, + help_text=mark_safe('Do not include Area Directors and WG Chairs; the system already tracks their availability.')) + timeranges = NameModelMultipleChoiceField( + widget=forms.CheckboxSelectMultiple, + required=False, + label=mark_safe('Times during which this WG can not meet:
Please explain any selections in Special Requests below.'), + queryset=TimerangeName.objects.all()) + adjacent_with_wg = forms.ChoiceField( + required=False, + label=mark_safe('Plan session adjacent with another WG:
(Immediately before or after another WG, no break in between, in the same room.)')) + send_notifications = forms.BooleanField(label="Send notification emails?", required=False, initial=False) + + def __init__(self, group, meeting, data=None, *args, **kwargs): + self.hidden = kwargs.pop('hidden', False) + self.notifications_optional = kwargs.pop('notifications_optional', False) + + self.group = group + formset_class = sessiondetailsformset_factory(max_num=3 if group.features.acts_like_wg else 50) + self.session_forms = formset_class(group=self.group, meeting=meeting, data=data) + super().__init__(data=data, *args, **kwargs) + if not self.notifications_optional: + self.fields['send_notifications'].widget = forms.HiddenInput() + + # Allow additional sessions for non-wg-like groups + if not self.group.features.acts_like_wg: + self.fields['num_session'].choices = ((n, str(n)) for n in range(1, 51)) + + self._add_widget_class(self.fields['third_session'].widget, 'form-check-input') + self.fields['comments'].widget = forms.Textarea(attrs={'rows': '3', 'cols': '65'}) + + other_groups = list(allowed_conflicting_groups().exclude(pk=group.pk).values_list('acronym', 'acronym').order_by('acronym')) + self.fields['adjacent_with_wg'].choices = [('', '--No preference')] + other_groups + group_acronym_choices = [('', '--Select WG(s)')] + other_groups + self.fields['joint_with_groups_selector'].choices = group_acronym_choices + + # Set up constraints for the meeting + self._wg_field_data = [] + for constraintname in meeting.group_conflict_types.all(): + # two fields for each constraint: a CharField for the group list and a selector to add entries + constraint_field = forms.CharField(max_length=255, required=False) + constraint_field.widget.attrs['data-slug'] = constraintname.slug + constraint_field.widget.attrs['data-constraint-name'] = str(constraintname).title() + constraint_field.widget.attrs['aria-label'] = f'{constraintname.slug}_input' + self._add_widget_class(constraint_field.widget, 'wg_constraint') + self._add_widget_class(constraint_field.widget, 'form-control') + + selector_field = forms.ChoiceField(choices=group_acronym_choices, required=False) + selector_field.widget.attrs['data-slug'] = constraintname.slug # used by onchange handler + self._add_widget_class(selector_field.widget, 'wg_constraint_selector') + self._add_widget_class(selector_field.widget, 'form-control') + + cfield_id = 'constraint_{}'.format(constraintname.slug) + cselector_id = 'wg_selector_{}'.format(constraintname.slug) + # keep an eye out for field name conflicts + log.assertion('cfield_id not in self.fields') + log.assertion('cselector_id not in self.fields') + self.fields[cfield_id] = constraint_field + self.fields[cselector_id] = selector_field + self._wg_field_data.append((constraintname, cfield_id, cselector_id)) + + # Show constraints that are not actually used by the meeting so these don't get lost + self._inactive_wg_field_data = [] + inactive_cnames = ConstraintName.objects.filter( + is_group_conflict=True # Only collect group conflicts... + ).exclude( + meeting=meeting # ...that are not enabled for this meeting... + ).filter( + constraint__source=group, # ...but exist for this group... + constraint__meeting=meeting, # ... at this meeting. + ).distinct() + + for inactive_constraint_name in inactive_cnames: + field_id = 'delete_{}'.format(inactive_constraint_name.slug) + self.fields[field_id] = forms.BooleanField(required=False, label='Delete this conflict', help_text='Delete this inactive conflict?') + self._add_widget_class(self.fields[field_id].widget, 'form-control') + constraints = group.constraint_source_set.filter(meeting=meeting, name=inactive_constraint_name) + self._inactive_wg_field_data.append( + (inactive_constraint_name, + ' '.join([c.target.acronym for c in constraints]), + field_id) + ) + + self.fields['joint_with_groups_selector'].widget.attrs['onchange'] = "document.form_post.joint_with_groups.value=document.form_post.joint_with_groups.value + ' ' + this.options[this.selectedIndex].value; return 1;" + self.fields["resources"].choices = [(x.pk, x.desc) for x in ResourceAssociation.objects.filter(name__used=True).order_by('name__order')] + + if self.hidden: + # replace all the widgets to start... + for key in list(self.fields.keys()): + self.fields[key].widget = forms.HiddenInput() + # re-replace a couple special cases + self.fields['resources'].widget = forms.MultipleHiddenInput() + self.fields['timeranges'].widget = forms.MultipleHiddenInput() + # and entirely replace bethere - no need to support searching if input is hidden + self.fields['bethere'] = ModelMultipleChoiceField( + widget=forms.MultipleHiddenInput, required=False, + queryset=Person.objects.all(), + ) + + def wg_constraint_fields(self): + """Iterates over wg constraint fields + + Intended for use in the template. + """ + for cname, cfield_id, cselector_id in self._wg_field_data: + yield cname, self[cfield_id], self[cselector_id] + + def wg_constraint_count(self): + """How many wg constraints are there?""" + return len(self._wg_field_data) + + def wg_constraint_field_ids(self): + """Iterates over wg constraint field IDs""" + for cname, cfield_id, _ in self._wg_field_data: + yield cname, cfield_id + + def inactive_wg_constraints(self): + for cname, value, field_id in self._inactive_wg_field_data: + yield cname, value, self[field_id] + + def inactive_wg_constraint_count(self): + return len(self._inactive_wg_field_data) + + def inactive_wg_constraint_field_ids(self): + """Iterates over wg constraint field IDs""" + for cname, _, field_id in self._inactive_wg_field_data: + yield cname, field_id + + @staticmethod + def _add_widget_class(widget, new_class): + """Add a new class, taking care in case some already exist""" + existing_classes = widget.attrs.get('class', '').split() + widget.attrs['class'] = ' '.join(existing_classes + [new_class]) + + def _join_conflicts(self, cleaned_data, slugs): + """Concatenate constraint fields from cleaned data into a single list""" + conflicts = [] + for cname, cfield_id, _ in self._wg_field_data: + if cname.slug in slugs and cfield_id in cleaned_data: + groups = cleaned_data[cfield_id] + # convert to python list (allow space or comma separated lists) + items = groups.replace(',', ' ').split() + conflicts.extend(items) + return conflicts + + def _validate_duplicate_conflicts(self, cleaned_data): + """Validate that no WGs appear in more than one constraint that does not allow duplicates + + Raises ValidationError + """ + # Only the older constraints (conflict, conflic2, conflic3) need to be mutually exclusive. + all_conflicts = self._join_conflicts(cleaned_data, ['conflict', 'conflic2', 'conflic3']) + seen = [] + duplicated = [] + errors = [] + for c in all_conflicts: + if c not in seen: + seen.append(c) + elif c not in duplicated: # only report once + duplicated.append(c) + errors.append(forms.ValidationError('%s appears in conflicts more than once' % c)) + return errors + + def clean_joint_with_groups(self): + groups = self.cleaned_data['joint_with_groups'] + check_conflict(groups, self.group) + return groups + + def clean_comments(self): + return clean_text_field(self.cleaned_data['comments']) + + def clean_bethere(self): + bethere = self.cleaned_data["bethere"] + if bethere: + extra = set( + Person.objects.filter( + role__group=self.group, role__name__in=["chair", "ad"] + ) + & bethere + ) + if extra: + extras = ", ".join(e.name for e in extra) + raise forms.ValidationError( + ( + f"Please remove the following person{pluralize(len(extra))}, the system " + f"tracks their availability due to their role{pluralize(len(extra))}: {extras}." + ) + ) + return bethere + + def clean_send_notifications(self): + return True if not self.notifications_optional else self.cleaned_data['send_notifications'] + + def is_valid(self): + return super().is_valid() and self.session_forms.is_valid() + + def clean(self): + super(SessionRequestForm, self).clean() + self.session_forms.clean() + + data = self.cleaned_data + + # Validate the individual conflict fields + for _, cfield_id, _ in self._wg_field_data: + try: + check_conflict(data[cfield_id], self.group) + except forms.ValidationError as e: + self.add_error(cfield_id, e) + + # Skip remaining tests if individual field tests had errors, + if self.errors: + return data + + # error if conflicts contain disallowed dupes + for error in self._validate_duplicate_conflicts(data): + self.add_error(None, error) + + # Verify expected number of session entries are present + num_sessions_with_data = len(self.session_forms.forms_to_keep) + num_sessions_expected = -1 + try: + num_sessions_expected = int(data.get('num_session', '')) + except ValueError: + self.add_error('num_session', 'Invalid value for number of sessions') + if num_sessions_with_data < num_sessions_expected: + self.add_error('num_session', 'Must provide data for all sessions') + + # if default (empty) option is selected, cleaned_data won't include num_session key + if num_sessions_expected != 2 and num_sessions_expected is not None: + if data.get('session_time_relation'): + self.add_error( + 'session_time_relation', + forms.ValidationError('Time between sessions can only be used when two sessions are requested.') + ) + + joint_session = data.get('joint_for_session', '') + if joint_session != '': + joint_session = int(joint_session) + if joint_session > num_sessions_with_data: + self.add_error( + 'joint_for_session', + forms.ValidationError( + f'Session {joint_session} can not be the joint session, the session has not been requested.' + ) + ) + + return data + + @property + def media(self): + # get media for our formset + return super().media + self.session_forms.media + forms.Media(js=('ietf/js/session_form.js',)) diff --git a/ietf/meeting/migrations/0016_alter_meeting_country_alter_meeting_time_zone.py b/ietf/meeting/migrations/0016_alter_meeting_country_alter_meeting_time_zone.py index 8f5db26112..8c467ea156 100644 --- a/ietf/meeting/migrations/0016_alter_meeting_country_alter_meeting_time_zone.py +++ b/ietf/meeting/migrations/0016_alter_meeting_country_alter_meeting_time_zone.py @@ -4,6 +4,7 @@ class Migration(migrations.Migration): + dependencies = [ ("meeting", "0015_alter_meeting_time_zone"), ] diff --git a/ietf/meeting/models.py b/ietf/meeting/models.py index de0192769e..7d9e318aab 100644 --- a/ietf/meeting/models.py +++ b/ietf/meeting/models.py @@ -149,7 +149,7 @@ def get_00_cutoff(self): cutoff_date = importantdate.date else: cutoff_date = self.date + datetime.timedelta(days=ImportantDateName.objects.get(slug='idcutoff').default_offset_days) - cutoff_time = datetime_from_date(cutoff_date, datetime.timezone.utc) + self.idsubmit_cutoff_time_utc + cutoff_time = datetime_from_date(cutoff_date, datetime.UTC) + self.idsubmit_cutoff_time_utc return cutoff_time def get_01_cutoff(self): @@ -161,7 +161,7 @@ def get_01_cutoff(self): cutoff_date = importantdate.date else: cutoff_date = self.date + datetime.timedelta(days=ImportantDateName.objects.get(slug='idcutoff').default_offset_days) - cutoff_time = datetime_from_date(cutoff_date, datetime.timezone.utc) + self.idsubmit_cutoff_time_utc + cutoff_time = datetime_from_date(cutoff_date, datetime.UTC) + self.idsubmit_cutoff_time_utc return cutoff_time def get_reopen_time(self): @@ -250,25 +250,39 @@ def get_attendance(self): # MeetingRegistration.attended started conflating badge-pickup and session attendance before IETF 114. # We've separated session attendance off to ietf.meeting.Attended, but need to report attendance at older # meetings correctly. - + # + # Looking up by registration and attendance records separately and joining in + # python is far faster than combining the Q objects in the query (~100x). + # Further optimization may be possible, but the queries are tricky... attended_per_meeting_registration = ( Q(registration__meeting=self) & ( Q(registration__attended=True) | Q(registration__checkedin=True) ) ) + attendees_by_reg = set( + Person.objects.filter(attended_per_meeting_registration).values_list( + "pk", flat=True + ) + ) + attended_per_meeting_attended = ( Q(attended__session__meeting=self) # Note that we are not filtering to plenary, wg, or rg sessions # as we do for nomcom eligibility - if picking up a badge (see above) # is good enough, just attending e.g. a training session is also good enough ) - attended = Person.objects.filter( - attended_per_meeting_registration | attended_per_meeting_attended - ).distinct() - - onsite = set(attended.filter(registration__meeting=self, registration__tickets__attendance_type__slug='onsite')) - remote = set(attended.filter(registration__meeting=self, registration__tickets__attendance_type__slug='remote')) + attendees_by_att = set( + Person.objects.filter(attended_per_meeting_attended).values_list( + "pk", flat=True + ) + ) + + attendees = Person.objects.filter( + pk__in=attendees_by_att | attendees_by_reg + ) + onsite = set(attendees.filter(registration__meeting=self, registration__tickets__attendance_type__slug='onsite')) + remote = set(attendees.filter(registration__meeting=self, registration__tickets__attendance_type__slug='remote')) remote.difference_update(onsite) return Attendance( @@ -942,8 +956,6 @@ class Meta: def __str__(self): return u"%s -> %s-%s" % (self.session, self.document.name, self.rev) -constraint_cache_uses = 0 -constraint_cache_initials = 0 class SessionQuerySet(models.QuerySet): def with_current_status(self): @@ -1172,7 +1184,7 @@ def can_manage_materials(self, user): return can_manage_materials(user,self.group) def is_material_submission_cutoff(self): - return date_today(datetime.timezone.utc) > self.meeting.get_submission_correction_date() + return date_today(datetime.UTC) > self.meeting.get_submission_correction_date() def joint_with_groups_acronyms(self): return [group.acronym for group in self.joint_with_groups.all()] diff --git a/ietf/meeting/resources.py b/ietf/meeting/resources.py index ede2b5b993..490b75f925 100644 --- a/ietf/meeting/resources.py +++ b/ietf/meeting/resources.py @@ -11,13 +11,23 @@ from ietf import api -from ietf.meeting.models import ( Meeting, ResourceAssociation, Constraint, Room, Schedule, Session, - TimeSlot, SchedTimeSessAssignment, SessionPresentation, FloorPlan, - UrlResource, ImportantDate, SlideSubmission, SchedulingEvent, - BusinessConstraint, ProceedingsMaterial, MeetingHost, Attended, - Registration, RegistrationTicket) +from ietf.meeting.models import (Meeting, ResourceAssociation, Constraint, Room, + Schedule, Session, + TimeSlot, SchedTimeSessAssignment, SessionPresentation, + FloorPlan, + UrlResource, ImportantDate, SlideSubmission, + SchedulingEvent, + BusinessConstraint, ProceedingsMaterial, MeetingHost, + Attended, + Registration, RegistrationTicket) + +from ietf.name.resources import ( + AttendanceTypeNameResource, + MeetingTypeNameResource, + RegistrationTicketTypeNameResource, +) + -from ietf.name.resources import MeetingTypeNameResource class MeetingResource(ModelResource): type = ToOneField(MeetingTypeNameResource, 'type') schedule = ToOneField('ietf.meeting.resources.ScheduleResource', 'schedule', null=True) @@ -433,11 +443,16 @@ class Meta: } api.meeting.register(AttendedResource()) -from ietf.meeting.resources import MeetingResource from ietf.person.resources import PersonResource class RegistrationResource(ModelResource): meeting = ToOneField(MeetingResource, 'meeting') person = ToOneField(PersonResource, 'person', null=True) + tickets = ToManyField( + 'ietf.meeting.resources.RegistrationTicketResource', + 'tickets', + full=True, + ) + class Meta: queryset = Registration.objects.all() serializer = api.Serializer() @@ -452,13 +467,17 @@ class Meta: "country_code": ALL, "email": ALL, "attended": ALL, + "checkedin": ALL, "meeting": ALL_WITH_RELATIONS, "person": ALL_WITH_RELATIONS, + "tickets": ALL_WITH_RELATIONS, } api.meeting.register(RegistrationResource()) class RegistrationTicketResource(ModelResource): registration = ToOneField(RegistrationResource, 'registration') + attendance_type = ToOneField(AttendanceTypeNameResource, 'attendance_type') + ticket_type = ToOneField(RegistrationTicketTypeNameResource, 'ticket_type') class Meta: queryset = RegistrationTicket.objects.all() serializer = api.Serializer() @@ -467,8 +486,8 @@ class Meta: ordering = ['id', ] filtering = { "id": ALL, - "ticket_type": ALL, - "attendance_type": ALL, + "ticket_type": ALL_WITH_RELATIONS, + "attendance_type": ALL_WITH_RELATIONS, "registration": ALL_WITH_RELATIONS, } api.meeting.register(RegistrationTicketResource()) diff --git a/ietf/meeting/tasks.py b/ietf/meeting/tasks.py index 784eb00d87..a73763560b 100644 --- a/ietf/meeting/tasks.py +++ b/ietf/meeting/tasks.py @@ -1,20 +1,77 @@ -# Copyright The IETF Trust 2024, All Rights Reserved +# Copyright The IETF Trust 2024-2026, All Rights Reserved # # Celery task definitions # -from celery import shared_task +import datetime + +from itertools import batched + +from celery import shared_task, chain +from django.db.models import IntegerField +from django.db.models.functions import Cast from django.utils import timezone from ietf.utils import log from .models import Meeting -from .utils import generate_proceedings_content +from .utils import ( + generate_proceedings_content, + resolve_materials_for_one_meeting, + store_blobs_for_one_meeting, +) from .views import generate_agenda_data from .utils import fetch_attendance_from_meetings +@shared_task +def agenda_data_refresh_task(num=None): + """Refresh agenda data for one plenary meeting + + If `num` is `None`, refreshes data for the current meeting. + """ + log.log( + f"Refreshing agenda data for {f"IETF-{num}" if num else "current IETF meeting"}" + ) + try: + generate_agenda_data(num, force_refresh=True) + except Exception as err: + # Log and swallow exceptions so failure on one meeting won't break a chain of + # tasks. This is used by agenda_data_refresh_all_task(). + log.log(f"ERROR: Refreshing agenda data failed for num={num}: {err}") + + @shared_task def agenda_data_refresh(): - generate_agenda_data(force_refresh=True) + """Deprecated. Use agenda_data_refresh_task() instead. + + TODO remove this after switching the periodic task to the new name + """ + log.log("Deprecated agenda_data_refresh task called!") + agenda_data_refresh_task() + + +@shared_task +def agenda_data_refresh_all_task(*, batch_size=10): + """Refresh agenda data for all plenary meetings + + Executes as a chain of tasks, each computing up to `batch_size` meetings + in a single task. + """ + meeting_numbers = sorted( + Meeting.objects.annotate( + number_as_int=Cast("number", output_field=IntegerField()) + ) + .filter(type_id="ietf", number_as_int__gt=64) + .values_list("number_as_int", flat=True) + ) + # Batch using chained maps rather than celery.chunk so we only use one worker + # at a time. + batched_task_chain = chain( + *( + agenda_data_refresh_task.map(nums) + for nums in batched(meeting_numbers, batch_size) + ) + ) + batched_task_chain.delay() @shared_task @@ -48,7 +105,9 @@ def proceedings_content_refresh_task(*, all=False): @shared_task def fetch_meeting_attendance_task(): # fetch most recent two meetings - meetings = Meeting.objects.filter(type="ietf", date__lte=timezone.now()).order_by("-date")[:2] + meetings = Meeting.objects.filter(type="ietf", date__lte=timezone.now()).order_by( + "-date" + )[:2] try: stats = fetch_attendance_from_meetings(meetings) except RuntimeError as err: @@ -57,7 +116,133 @@ def fetch_meeting_attendance_task(): for meeting, meeting_stats in zip(meetings, stats): log.log( "Fetched data for meeting {:>3}: {:4d} created, {:4d} updated, {:4d} deleted, {:4d} processed".format( - meeting.number, meeting_stats['created'], meeting_stats['updated'], meeting_stats['deleted'], - meeting_stats['processed'] + meeting.number, + meeting_stats["created"], + meeting_stats["updated"], + meeting_stats["deleted"], + meeting_stats["processed"], ) ) + + +def _select_meetings( + meetings: list[str] | None = None, + meetings_since: str | None = None, + meetings_until: str | None = None, +): # nyah + """Select meetings by number or date range""" + # IETF-1 = 1986-01-16 + EARLIEST_MEETING_DATE = datetime.datetime(1986, 1, 1) + meetings_since_dt: datetime.datetime | None = None + meetings_until_dt: datetime.datetime | None = None + + if meetings_since == "zero": + meetings_since_dt = EARLIEST_MEETING_DATE + elif meetings_since is not None: + try: + meetings_since_dt = datetime.datetime.fromisoformat(meetings_since) + except ValueError: + log.log( + "Failed to parse meetings_since='{meetings_since}' with fromisoformat" + ) + raise + + if meetings_until is not None: + try: + meetings_until_dt = datetime.datetime.fromisoformat(meetings_until) + except ValueError: + log.log( + "Failed to parse meetings_until='{meetings_until}' with fromisoformat" + ) + raise + if meetings_since_dt is None: + # if we only got meetings_until, start from the first meeting + meetings_since_dt = EARLIEST_MEETING_DATE + + if meetings is None: + if meetings_since_dt is None: + log.log("No meetings requested, doing nothing.") + return Meeting.objects.none() + meetings_qs = Meeting.objects.filter(date__gte=meetings_since_dt) + if meetings_until_dt is not None: + meetings_qs = meetings_qs.filter(date__lte=meetings_until_dt) + log.log( + "Selecting meetings between " + f"{meetings_since_dt} and {meetings_until_dt}" + ) + else: + log.log(f"Selecting meetings since {meetings_since_dt}") + else: + if meetings_since_dt is not None: + log.log( + "Ignoring meetings_since and meetings_until " + "because specific meetings were requested." + ) + meetings_qs = Meeting.objects.filter(number__in=meetings) + return meetings_qs + + +@shared_task +def resolve_meeting_materials_task( + *, # only allow kw arguments + meetings: list[str] | None = None, + meetings_since: str | None = None, + meetings_until: str | None = None, +): + """Run materials resolver on meetings + + Can request a set of meetings by number by passing a list in the meetings arg, or + by range by passing an iso-format timestamps in meetings_since / meetings_until. + To select all meetings, set meetings_since="zero" and omit other parameters. + """ + meetings_qs = _select_meetings(meetings, meetings_since, meetings_until) + for meeting in meetings_qs.order_by("date"): + log.log( + f"Resolving materials for {meeting.type_id} " + f"meeting {meeting.number} ({meeting.date})..." + ) + mark = timezone.now() + try: + resolve_materials_for_one_meeting(meeting) + except Exception as err: + log.log( + "Exception raised while resolving materials for " + f"meeting {meeting.number}: {err}" + ) + else: + log.log( + f"Resolved in {(timezone.now() - mark).total_seconds():0.3f} seconds." + ) + + +@shared_task +def store_meeting_materials_as_blobs_task( + *, # only allow kw arguments + meetings: list[str] | None = None, + meetings_since: str | None = None, + meetings_until: str | None = None, +): + """Push meeting materials into the blob store + + Can request a set of meetings by number by passing a list in the meetings arg, or + by range by passing an iso-format timestamps in meetings_since / meetings_until. + To select all meetings, set meetings_since="zero" and omit other parameters. + """ + meetings_qs = _select_meetings(meetings, meetings_since, meetings_until) + for meeting in meetings_qs.order_by("date"): + log.log( + f"Creating blobs for materials for {meeting.type_id} " + f"meeting {meeting.number} ({meeting.date})..." + ) + mark = timezone.now() + try: + store_blobs_for_one_meeting(meeting) + except Exception as err: + log.log( + "Exception raised while creating blobs for " + f"meeting {meeting.number}: {err}" + ) + else: + log.log( + f"Blobs created in {(timezone.now() - mark).total_seconds():0.3f} seconds." + ) diff --git a/ietf/secr/sreq/templatetags/ams_filters.py b/ietf/meeting/templatetags/ams_filters.py similarity index 96% rename from ietf/secr/sreq/templatetags/ams_filters.py rename to ietf/meeting/templatetags/ams_filters.py index 3ef872232a..a8175a81d6 100644 --- a/ietf/secr/sreq/templatetags/ams_filters.py +++ b/ietf/meeting/templatetags/ams_filters.py @@ -1,3 +1,5 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + from django import template from ietf.person.models import Person diff --git a/ietf/meeting/tests_js.py b/ietf/meeting/tests_js.py index a184a7c6d0..3269342924 100644 --- a/ietf/meeting/tests_js.py +++ b/ietf/meeting/tests_js.py @@ -1260,7 +1260,7 @@ def _assert_ietf_tz_correct(meetings, tz): # to inherit Django's settings.TIME_ZONE but I don't know whether that's guaranteed to be consistent. # To avoid test fragility, ask Moment what it considers local and expect that. local_tz = self.driver.execute_script('return moment.tz.guess();') - local_tz_opt = tz_select_input.find_element(By.CSS_SELECTOR, 'option[value=%s]' % local_tz) + local_tz_opt = tz_select_input.find_element(By.CSS_SELECTOR, 'option[value="%s"]' % local_tz) local_tz_bottom_opt = tz_select_bottom_input.find_element(By.CSS_SELECTOR, 'option[value="%s"]' % local_tz) # Should start off in local time zone @@ -1576,7 +1576,7 @@ def test_delete_timeslot_cancel(self): def do_delete_time_interval_test(self, cancel=False): delete_time_local = datetime_from_date(self.meeting.date, self.meeting.tz()).replace(hour=10) - delete_time = delete_time_local.astimezone(datetime.timezone.utc) + delete_time = delete_time_local.astimezone(datetime.UTC) duration = datetime.timedelta(minutes=60) delete: [TimeSlot] = TimeSlotFactory.create_batch( # type: ignore[annotation-unchecked] diff --git a/ietf/secr/sreq/tests.py b/ietf/meeting/tests_session_requests.py similarity index 84% rename from ietf/secr/sreq/tests.py rename to ietf/meeting/tests_session_requests.py index 847b993e1c..0cb092d2f8 100644 --- a/ietf/secr/sreq/tests.py +++ b/ietf/meeting/tests_session_requests.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2013-2022, All Rights Reserved +# Copyright The IETF Trust 2013-2025, All Rights Reserved # -*- coding: utf-8 -*- @@ -15,30 +15,15 @@ from ietf.name.models import ConstraintName, TimerangeName from ietf.person.factories import PersonFactory from ietf.person.models import Person -from ietf.secr.sreq.forms import SessionForm +from ietf.meeting.forms import SessionRequestForm from ietf.utils.mail import outbox, empty_outbox, get_payload_text, send_mail from ietf.utils.timezone import date_today from pyquery import PyQuery -SECR_USER='secretary' +SECR_USER = 'secretary' -class SreqUrlTests(TestCase): - def test_urls(self): - MeetingFactory(type_id='ietf',date=date_today()) - - self.client.login(username="secretary", password="secretary+password") - - r = self.client.get("/secr/") - self.assertEqual(r.status_code, 200) - - r = self.client.get("/secr/sreq/") - self.assertEqual(r.status_code, 200) - - testgroup=GroupFactory() - r = self.client.get("/secr/sreq/%s/new/" % testgroup.acronym) - self.assertEqual(r.status_code, 200) class SessionRequestTestCase(TestCase): def test_main(self): @@ -46,7 +31,7 @@ def test_main(self): SessionFactory.create_batch(2, meeting=meeting, status_id='sched') SessionFactory.create_batch(2, meeting=meeting, status_id='disappr') # Several unscheduled groups come from make_immutable_base_data - url = reverse('ietf.secr.sreq.views.main') + url = reverse('ietf.meeting.views_session_request.list_view') self.client.login(username="secretary", password="secretary+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) @@ -62,27 +47,27 @@ def test_approve(self): mars = GroupFactory(parent=area, acronym='mars') # create session waiting for approval session = SessionFactory(meeting=meeting, group=mars, status_id='apprw') - url = reverse('ietf.secr.sreq.views.approve', kwargs={'acronym':'mars'}) + url = reverse('ietf.meeting.views_session_request.approve_request', kwargs={'acronym': 'mars'}) self.client.login(username="ad", password="ad+password") r = self.client.get(url) - self.assertRedirects(r,reverse('ietf.secr.sreq.views.view', kwargs={'acronym':'mars'})) + self.assertRedirects(r, reverse('ietf.meeting.views_session_request.view_request', kwargs={'acronym': 'mars'})) self.assertEqual(SchedulingEvent.objects.filter(session=session).order_by('-id')[0].status_id, 'appr') - + def test_cancel(self): meeting = MeetingFactory(type_id='ietf', date=date_today()) ad = Person.objects.get(user__username='ad') area = RoleFactory(name_id='ad', person=ad, group__type_id='area').group session = SessionFactory(meeting=meeting, group__parent=area, group__acronym='mars', status_id='sched') - url = reverse('ietf.secr.sreq.views.cancel', kwargs={'acronym':'mars'}) + url = reverse('ietf.meeting.views_session_request.cancel_request', kwargs={'acronym': 'mars'}) self.client.login(username="ad", password="ad+password") r = self.client.get(url) - self.assertRedirects(r,reverse('ietf.secr.sreq.views.main')) + self.assertRedirects(r, reverse('ietf.meeting.views_session_request.list_view')) self.assertEqual(SchedulingEvent.objects.filter(session=session).order_by('-id')[0].status_id, 'deleted') def test_cancel_notification_msg(self): to = "" subject = "Dummy subject" - template = "sreq/session_cancel_notification.txt" + template = "meeting/session_cancel_notification.txt" meeting = MeetingFactory(type_id="ietf", date=date_today()) requester = PersonFactory(name="James O'Rourke", user__username="jimorourke") context = {"meeting": meeting, "requester": requester} @@ -113,9 +98,9 @@ def test_edit(self): group4 = GroupFactory() iabprog = GroupFactory(type_id='program') - SessionFactory(meeting=meeting,group=mars,status_id='sched') + SessionFactory(meeting=meeting, group=mars, status_id='sched') - url = reverse('ietf.secr.sreq.views.edit', kwargs={'acronym':'mars'}) + url = reverse('ietf.meeting.views_session_request.edit_request', kwargs={'acronym': 'mars'}) self.client.login(username="marschairman", password="marschairman+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) @@ -123,9 +108,9 @@ def test_edit(self): comments = 'need lights' mars_sessions = meeting.session_set.filter(group__acronym='mars') empty_outbox() - post_data = {'num_session':'2', + post_data = {'num_session': '2', 'attendees': attendees, - 'constraint_chair_conflict':iabprog.acronym, + 'constraint_chair_conflict': iabprog.acronym, 'session_time_relation': 'subsequent-days', 'adjacent_with_wg': group2.acronym, 'joint_with_groups': group3.acronym + ' ' + group4.acronym, @@ -135,7 +120,7 @@ def test_edit(self): 'session_set-INITIAL_FORMS': '1', 'session_set-MIN_NUM_FORMS': '1', 'session_set-MAX_NUM_FORMS': '3', - 'session_set-0-id':mars_sessions[0].pk, + 'session_set-0-id': mars_sessions[0].pk, 'session_set-0-name': mars_sessions[0].name, 'session_set-0-short': mars_sessions[0].short, 'session_set-0-purpose': mars_sessions[0].purpose_id, @@ -169,7 +154,7 @@ def test_edit(self): 'session_set-2-DELETE': 'on', 'submit': 'Continue'} r = self.client.post(url, post_data, HTTP_HOST='example.com') - redirect_url = reverse('ietf.secr.sreq.views.view', kwargs={'acronym': 'mars'}) + redirect_url = reverse('ietf.meeting.views_session_request.view_request', kwargs={'acronym': 'mars'}) self.assertRedirects(r, redirect_url) # Check whether updates were stored in the database @@ -204,17 +189,17 @@ def test_edit(self): # Edit again, changing the joint sessions and clearing some fields. The behaviour of # edit is different depending on whether previous joint sessions were recorded. empty_outbox() - post_data = {'num_session':'2', - 'attendees':attendees, - 'constraint_chair_conflict':'', - 'comments':'need lights', + post_data = {'num_session': '2', + 'attendees': attendees, + 'constraint_chair_conflict': '', + 'comments': 'need lights', 'joint_with_groups': group2.acronym, 'joint_for_session': '1', 'session_set-TOTAL_FORMS': '3', # matches what view actually sends, even with only 2 filled in 'session_set-INITIAL_FORMS': '2', 'session_set-MIN_NUM_FORMS': '1', 'session_set-MAX_NUM_FORMS': '3', - 'session_set-0-id':sessions[0].pk, + 'session_set-0-id': sessions[0].pk, 'session_set-0-name': sessions[0].name, 'session_set-0-short': sessions[0].short, 'session_set-0-purpose': sessions[0].purpose_id, @@ -270,7 +255,6 @@ def test_edit(self): r = self.client.get(redirect_url) self.assertContains(r, 'First session with: {}'.format(group2.acronym)) - def test_edit_constraint_bethere(self): meeting = MeetingFactory(type_id='ietf', date=date_today()) mars = RoleFactory(name_id='chair', person__user__username='marschairman', group__acronym='mars').group @@ -282,7 +266,7 @@ def test_edit_constraint_bethere(self): name_id='bethere', ) self.assertEqual(session.people_constraints.count(), 1) - url = reverse('ietf.secr.sreq.views.edit', kwargs=dict(acronym='mars')) + url = reverse('ietf.meeting.views_session_request.edit_request', kwargs=dict(acronym='mars')) self.client.login(username='marschairman', password='marschairman+password') attendees = '10' ad = Person.objects.get(user__username='ad') @@ -290,8 +274,8 @@ def test_edit_constraint_bethere(self): 'num_session': '1', 'attendees': attendees, 'bethere': str(ad.pk), - 'constraint_chair_conflict':'', - 'comments':'', + 'constraint_chair_conflict': '', + 'comments': '', 'joint_with_groups': '', 'joint_for_session': '', 'delete_conflict': 'on', @@ -299,7 +283,7 @@ def test_edit_constraint_bethere(self): 'session_set-INITIAL_FORMS': '1', 'session_set-MIN_NUM_FORMS': '1', 'session_set-MAX_NUM_FORMS': '3', - 'session_set-0-id':session.pk, + 'session_set-0-id': session.pk, 'session_set-0-name': session.name, 'session_set-0-short': session.short, 'session_set-0-purpose': session.purpose_id, @@ -313,8 +297,8 @@ def test_edit_constraint_bethere(self): 'session_set-1-id': '', 'session_set-1-name': '', 'session_set-1-short': '', - 'session_set-1-purpose':'regular', - 'session_set-1-type':'regular', + 'session_set-1-purpose': 'regular', + 'session_set-1-type': 'regular', 'session_set-1-requested_duration': '', 'session_set-1-on_agenda': 'True', 'session_set-1-attendees': attendees, @@ -333,7 +317,7 @@ def test_edit_constraint_bethere(self): 'submit': 'Save', } r = self.client.post(url, post_data, HTTP_HOST='example.com') - redirect_url = reverse('ietf.secr.sreq.views.view', kwargs={'acronym': 'mars'}) + redirect_url = reverse('ietf.meeting.views_session_request.view_request', kwargs={'acronym': 'mars'}) self.assertRedirects(r, redirect_url) self.assertEqual([pc.person for pc in session.people_constraints.all()], [ad]) @@ -350,7 +334,7 @@ def test_edit_inactive_conflicts(self): target=other_group, ) - url = reverse('ietf.secr.sreq.views.edit', kwargs=dict(acronym='mars')) + url = reverse('ietf.meeting.views_session_request.edit_request', kwargs=dict(acronym='mars')) self.client.login(username='marschairman', password='marschairman+password') r = self.client.get(url) self.assertEqual(r.status_code, 200) @@ -360,17 +344,17 @@ def test_edit_inactive_conflicts(self): found = q('input#id_delete_conflict[type="checkbox"]') self.assertEqual(len(found), 1) delete_checkbox = found[0] - # check that the label on the checkbox is correct - self.assertIn('Delete this conflict', delete_checkbox.tail) + self.assertIn('Delete this conflict', delete_checkbox.label.text) # check that the target is displayed correctly in the UI - self.assertIn(other_group.acronym, delete_checkbox.find('../input[@type="text"]').value) + row = found.parent().parent() + self.assertIn(other_group.acronym, row.find('input[@type="text"]').val()) attendees = '10' post_data = { 'num_session': '1', 'attendees': attendees, - 'constraint_chair_conflict':'', - 'comments':'', + 'constraint_chair_conflict': '', + 'comments': '', 'joint_with_groups': '', 'joint_for_session': '', 'delete_conflict': 'on', @@ -378,7 +362,7 @@ def test_edit_inactive_conflicts(self): 'session_set-INITIAL_FORMS': '1', 'session_set-MIN_NUM_FORMS': '1', 'session_set-MAX_NUM_FORMS': '3', - 'session_set-0-id':session.pk, + 'session_set-0-id': session.pk, 'session_set-0-name': session.name, 'session_set-0-short': session.short, 'session_set-0-purpose': session.purpose_id, @@ -392,28 +376,28 @@ def test_edit_inactive_conflicts(self): 'submit': 'Save', } r = self.client.post(url, post_data, HTTP_HOST='example.com') - redirect_url = reverse('ietf.secr.sreq.views.view', kwargs={'acronym': 'mars'}) + redirect_url = reverse('ietf.meeting.views_session_request.view_request', kwargs={'acronym': 'mars'}) self.assertRedirects(r, redirect_url) self.assertEqual(len(mars.constraint_source_set.filter(name_id='conflict')), 0) def test_tool_status(self): MeetingFactory(type_id='ietf', date=date_today()) - url = reverse('ietf.secr.sreq.views.tool_status') + url = reverse('ietf.meeting.views_session_request.status') self.client.login(username="secretary", password="secretary+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) - r = self.client.post(url, {'message':'locked', 'submit':'Lock'}) - self.assertRedirects(r,reverse('ietf.secr.sreq.views.main')) + r = self.client.post(url, {'message': 'locked', 'submit': 'Lock'}) + self.assertRedirects(r, reverse('ietf.meeting.views_session_request.list_view')) def test_new_req_constraint_types(self): """Configurable constraint types should be handled correctly in a new request - Relies on SessionForm representing constraint values with element IDs + Relies on SessionRequestForm representing constraint values with element IDs like id_constraint_ """ meeting = MeetingFactory(type_id='ietf', date=date_today()) RoleFactory(name_id='chair', person__user__username='marschairman', group__acronym='mars') - url = reverse('ietf.secr.sreq.views.new', kwargs=dict(acronym='mars')) + url = reverse('ietf.meeting.views_session_request.new_request', kwargs=dict(acronym='mars')) self.client.login(username="marschairman", password="marschairman+password") for expected in [ @@ -441,7 +425,7 @@ def test_edit_req_constraint_types(self): add_to_schedule=False) RoleFactory(name_id='chair', person__user__username='marschairman', group__acronym='mars') - url = reverse('ietf.secr.sreq.views.edit', kwargs=dict(acronym='mars')) + url = reverse('ietf.meeting.views_session_request.edit_request', kwargs=dict(acronym='mars')) self.client.login(username='marschairman', password='marschairman+password') for expected in [ @@ -460,6 +444,7 @@ def test_edit_req_constraint_types(self): ['id_constraint_{}'.format(conf_name) for conf_name in expected], ) + class SubmitRequestCase(TestCase): def setUp(self): super(SubmitRequestCase, self).setUp() @@ -476,15 +461,15 @@ def test_submit_request(self): group3 = GroupFactory(parent=area) group4 = GroupFactory(parent=area) session_count_before = Session.objects.filter(meeting=meeting, group=group).count() - url = reverse('ietf.secr.sreq.views.new',kwargs={'acronym':group.acronym}) - confirm_url = reverse('ietf.secr.sreq.views.confirm',kwargs={'acronym':group.acronym}) - main_url = reverse('ietf.secr.sreq.views.main') + url = reverse('ietf.meeting.views_session_request.new_request', kwargs={'acronym': group.acronym}) + confirm_url = reverse('ietf.meeting.views_session_request.confirm', kwargs={'acronym': group.acronym}) + main_url = reverse('ietf.meeting.views_session_request.list_view') attendees = '10' comments = 'need projector' - post_data = {'num_session':'1', - 'attendees':attendees, - 'constraint_chair_conflict':'', - 'comments':comments, + post_data = {'num_session': '1', + 'attendees': attendees, + 'constraint_chair_conflict': '', + 'comments': comments, 'adjacent_with_wg': group2.acronym, 'timeranges': ['thursday-afternoon-early', 'thursday-afternoon-late'], 'joint_with_groups': group3.acronym + ' ' + group4.acronym, @@ -506,7 +491,7 @@ def test_submit_request(self): 'session_set-0-DELETE': '', 'submit': 'Continue'} self.client.login(username="secretary", password="secretary+password") - r = self.client.post(url,post_data) + r = self.client.post(url, post_data) self.assertEqual(r.status_code, 200) # Verify the contents of the confirm view @@ -515,13 +500,13 @@ def test_submit_request(self): self.assertContains(r, 'First session with: {} {}'.format(group3.acronym, group4.acronym)) post_data['submit'] = 'Submit' - r = self.client.post(confirm_url,post_data) + r = self.client.post(confirm_url, post_data) self.assertRedirects(r, main_url) session_count_after = Session.objects.filter(meeting=meeting, group=group, type='regular').count() self.assertEqual(session_count_after, session_count_before + 1) # test that second confirm does not add sessions - r = self.client.post(confirm_url,post_data) + r = self.client.post(confirm_url, post_data) self.assertRedirects(r, main_url) session_count_after = Session.objects.filter(meeting=meeting, group=group, type='regular').count() self.assertEqual(session_count_after, session_count_before + 1) @@ -535,42 +520,6 @@ def test_submit_request(self): ) self.assertEqual(set(list(session.joint_with_groups.all())), set([group3, group4])) - def test_submit_request_invalid(self): - MeetingFactory(type_id='ietf', date=date_today()) - ad = Person.objects.get(user__username='ad') - area = RoleFactory(name_id='ad', person=ad, group__type_id='area').group - group = GroupFactory(parent=area) - url = reverse('ietf.secr.sreq.views.new',kwargs={'acronym':group.acronym}) - attendees = '10' - comments = 'need projector' - post_data = { - 'num_session':'2', - 'attendees':attendees, - 'constraint_chair_conflict':'', - 'comments':comments, - 'session_set-TOTAL_FORMS': '1', - 'session_set-INITIAL_FORMS': '1', - 'session_set-MIN_NUM_FORMS': '1', - 'session_set-MAX_NUM_FORMS': '3', - # no 'session_set-0-id' to create a new session - 'session_set-0-name': '', - 'session_set-0-short': '', - 'session_set-0-purpose': 'regular', - 'session_set-0-type': 'regular', - 'session_set-0-requested_duration': '3600', - 'session_set-0-on_agenda': True, - 'session_set-0-remote_instructions': '', - 'session_set-0-attendees': attendees, - 'session_set-0-comments': comments, - 'session_set-0-DELETE': '', - } - self.client.login(username="secretary", password="secretary+password") - r = self.client.post(url,post_data) - self.assertEqual(r.status_code, 200) - q = PyQuery(r.content) - self.assertEqual(len(q('#session-request-form')),1) - self.assertContains(r, 'Must provide data for all sessions') - def test_submit_request_check_constraints(self): m1 = MeetingFactory(type_id='ietf', date=date_today() - datetime.timedelta(days=100)) MeetingFactory(type_id='ietf', date=date_today(), @@ -597,7 +546,7 @@ def test_submit_request_check_constraints(self): self.client.login(username="secretary", password="secretary+password") - url = reverse('ietf.secr.sreq.views.new',kwargs={'acronym':group.acronym}) + url = reverse('ietf.meeting.views_session_request.new_request', kwargs={'acronym': group.acronym}) r = self.client.get(url + '?previous') self.assertEqual(r.status_code, 200) q = PyQuery(r.content) @@ -607,11 +556,11 @@ def test_submit_request_check_constraints(self): attendees = '10' comments = 'need projector' - post_data = {'num_session':'1', - 'attendees':attendees, + post_data = {'num_session': '1', + 'attendees': attendees, 'constraint_chair_conflict': group.acronym, - 'comments':comments, - 'session_set-TOTAL_FORMS': '1', + 'comments': comments, + 'session_set-TOTAL_FORMS': '3', 'session_set-INITIAL_FORMS': '1', 'session_set-MIN_NUM_FORMS': '1', 'session_set-MAX_NUM_FORMS': '3', @@ -626,11 +575,31 @@ def test_submit_request_check_constraints(self): 'session_set-0-attendees': attendees, 'session_set-0-comments': comments, 'session_set-0-DELETE': '', + 'session_set-1-name': '', + 'session_set-1-short': '', + 'session_set-1-purpose': session.purpose_id, + 'session_set-1-type': session.type_id, + 'session_set-1-requested_duration': '', + 'session_set-1-on_agenda': session.on_agenda, + 'session_set-1-remote_instructions': '', + 'session_set-1-attendees': attendees, + 'session_set-1-comments': '', + 'session_set-1-DELETE': 'on', + 'session_set-2-name': '', + 'session_set-2-short': '', + 'session_set-2-purpose': session.purpose_id, + 'session_set-2-type': session.type_id, + 'session_set-2-requested_duration': '', + 'session_set-2-on_agenda': session.on_agenda, + 'session_set-2-remote_instructions': '', + 'session_set-2-attendees': attendees, + 'session_set-2-comments': '', + 'session_set-2-DELETE': 'on', 'submit': 'Continue'} - r = self.client.post(url,post_data) + r = self.client.post(url, post_data) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertEqual(len(q('#session-request-form')),1) + self.assertEqual(len(q('#session-request-form')), 1) self.assertContains(r, "Cannot declare a conflict with the same group") def test_request_notification(self): @@ -645,18 +614,18 @@ def test_request_notification(self): RoleFactory(name_id='chair', group=group, person__user__username='ameschairman') resource = ResourceAssociation.objects.create(name_id='project') # Bit of a test data hack - the fixture now has no used resources to pick from - resource.name.used=True + resource.name.used = True resource.name.save() - url = reverse('ietf.secr.sreq.views.new',kwargs={'acronym':group.acronym}) - confirm_url = reverse('ietf.secr.sreq.views.confirm',kwargs={'acronym':group.acronym}) + url = reverse('ietf.meeting.views_session_request.new_request', kwargs={'acronym': group.acronym}) + confirm_url = reverse('ietf.meeting.views_session_request.confirm', kwargs={'acronym': group.acronym}) len_before = len(outbox) attendees = '10' - post_data = {'num_session':'2', - 'attendees':attendees, - 'bethere':str(ad.pk), - 'constraint_chair_conflict':group4.acronym, - 'comments':'', + post_data = {'num_session': '2', + 'attendees': attendees, + 'bethere': str(ad.pk), + 'constraint_chair_conflict': group4.acronym, + 'comments': '', 'resources': resource.pk, 'session_time_relation': 'subsequent-days', 'adjacent_with_wg': group2.acronym, @@ -692,23 +661,23 @@ def test_request_notification(self): 'submit': 'Continue'} self.client.login(username="ameschairman", password="ameschairman+password") # submit - r = self.client.post(url,post_data) + r = self.client.post(url, post_data) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertTrue('Confirm' in str(q("title")), r.context['form'].errors) # confirm post_data['submit'] = 'Submit' - r = self.client.post(confirm_url,post_data) - self.assertRedirects(r, reverse('ietf.secr.sreq.views.main')) - self.assertEqual(len(outbox),len_before+1) + r = self.client.post(confirm_url, post_data) + self.assertRedirects(r, reverse('ietf.meeting.views_session_request.list_view')) + self.assertEqual(len(outbox), len_before + 1) notification = outbox[-1] notification_payload = get_payload_text(notification) - sessions = Session.objects.filter(meeting=meeting,group=group) + sessions = Session.objects.filter(meeting=meeting, group=group) self.assertEqual(len(sessions), 2) session = sessions[0] - self.assertEqual(session.resources.count(),1) - self.assertEqual(session.people_constraints.count(),1) + self.assertEqual(session.resources.count(), 1) + self.assertEqual(session.people_constraints.count(), 1) self.assertEqual(session.constraints().get(name='time_relation').time_relation, 'subsequent-days') self.assertEqual(session.constraints().get(name='wg_adjacent').target.acronym, group2.acronym) self.assertEqual( @@ -731,7 +700,7 @@ def test_request_notification(self): def test_request_notification_msg(self): to = "" subject = "Dummy subject" - template = "sreq/session_request_notification.txt" + template = "meeting/session_request_notification.txt" header = "A new" meeting = MeetingFactory(type_id="ietf", date=date_today()) requester = PersonFactory(name="James O'Rourke", user__username="jimorourke") @@ -767,19 +736,19 @@ def test_request_notification_third_session(self): RoleFactory(name_id='chair', group=group, person__user__username='ameschairman') resource = ResourceAssociation.objects.create(name_id='project') # Bit of a test data hack - the fixture now has no used resources to pick from - resource.name.used=True + resource.name.used = True resource.name.save() - url = reverse('ietf.secr.sreq.views.new',kwargs={'acronym':group.acronym}) - confirm_url = reverse('ietf.secr.sreq.views.confirm',kwargs={'acronym':group.acronym}) + url = reverse('ietf.meeting.views_session_request.new_request', kwargs={'acronym': group.acronym}) + confirm_url = reverse('ietf.meeting.views_session_request.confirm', kwargs={'acronym': group.acronym}) len_before = len(outbox) attendees = '10' - post_data = {'num_session':'2', + post_data = {'num_session': '2', 'third_session': 'true', - 'attendees':attendees, - 'bethere':str(ad.pk), - 'constraint_chair_conflict':group4.acronym, - 'comments':'', + 'attendees': attendees, + 'bethere': str(ad.pk), + 'constraint_chair_conflict': group4.acronym, + 'comments': '', 'resources': resource.pk, 'session_time_relation': 'subsequent-days', 'adjacent_with_wg': group2.acronym, @@ -826,23 +795,23 @@ def test_request_notification_third_session(self): 'submit': 'Continue'} self.client.login(username="ameschairman", password="ameschairman+password") # submit - r = self.client.post(url,post_data) + r = self.client.post(url, post_data) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertTrue('Confirm' in str(q("title")), r.context['form'].errors) # confirm post_data['submit'] = 'Submit' - r = self.client.post(confirm_url,post_data) - self.assertRedirects(r, reverse('ietf.secr.sreq.views.main')) - self.assertEqual(len(outbox),len_before+1) + r = self.client.post(confirm_url, post_data) + self.assertRedirects(r, reverse('ietf.meeting.views_session_request.list_view')) + self.assertEqual(len(outbox), len_before + 1) notification = outbox[-1] notification_payload = get_payload_text(notification) - sessions = Session.objects.filter(meeting=meeting,group=group) + sessions = Session.objects.filter(meeting=meeting, group=group) self.assertEqual(len(sessions), 3) session = sessions[0] - self.assertEqual(session.resources.count(),1) - self.assertEqual(session.people_constraints.count(),1) + self.assertEqual(session.resources.count(), 1) + self.assertEqual(session.people_constraints.count(), 1) self.assertEqual(session.constraints().get(name='time_relation').time_relation, 'subsequent-days') self.assertEqual(session.constraints().get(name='wg_adjacent').target.acronym, group2.acronym) self.assertEqual( @@ -861,16 +830,17 @@ def test_request_notification_third_session(self): self.assertIn('1 Hour, 1 Hour, 1 Hour', notification_payload) self.assertIn('The third session requires your approval', notification_payload) + class LockAppTestCase(TestCase): def setUp(self): super().setUp() - self.meeting = MeetingFactory(type_id='ietf', date=date_today(),session_request_lock_message='locked') + self.meeting = MeetingFactory(type_id='ietf', date=date_today(), session_request_lock_message='locked') self.group = GroupFactory(acronym='mars') RoleFactory(name_id='chair', group=self.group, person__user__username='marschairman') - SessionFactory(group=self.group,meeting=self.meeting) + SessionFactory(group=self.group, meeting=self.meeting) def test_edit_request(self): - url = reverse('ietf.secr.sreq.views.edit',kwargs={'acronym':self.group.acronym}) + url = reverse('ietf.meeting.views_session_request.edit_request', kwargs={'acronym': self.group.acronym}) self.client.login(username="secretary", password="secretary+password") r = self.client.get(url) self.assertEqual(r.status_code, 200) @@ -882,48 +852,49 @@ def test_edit_request(self): self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertEqual(len(q(':disabled[name="submit"]')), 1) - + def test_view_request(self): - url = reverse('ietf.secr.sreq.views.view',kwargs={'acronym':self.group.acronym}) + url = reverse('ietf.meeting.views_session_request.view_request', kwargs={'acronym': self.group.acronym}) self.client.login(username="secretary", password="secretary+password") - r = self.client.get(url,follow=True) + r = self.client.get(url, follow=True) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertEqual(len(q(':enabled[name="edit"]')), 1) # secretary can edit chair = self.group.role_set.filter(name_id='chair').first().person.user.username self.client.login(username=chair, password=f'{chair}+password') - r = self.client.get(url,follow=True) + r = self.client.get(url, follow=True) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) self.assertEqual(len(q(':disabled[name="edit"]')), 1) # chair cannot edit def test_new_request(self): - url = reverse('ietf.secr.sreq.views.new',kwargs={'acronym':self.group.acronym}) - + url = reverse('ietf.meeting.views_session_request.new_request', kwargs={'acronym': self.group.acronym}) + # try as WG Chair self.client.login(username="marschairman", password="marschairman+password") r = self.client.get(url, follow=True) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertEqual(len(q('#session-request-form')),0) - + self.assertEqual(len(q('#session-request-form')), 0) + # try as Secretariat self.client.login(username="secretary", password="secretary+password") - r = self.client.get(url,follow=True) + r = self.client.get(url, follow=True) self.assertEqual(r.status_code, 200) q = PyQuery(r.content) - self.assertEqual(len(q('#session-request-form')),1) - + self.assertEqual(len(q('#session-request-form')), 1) + + class NotMeetingCase(TestCase): def test_not_meeting(self): - MeetingFactory(type_id='ietf',date=date_today()) + MeetingFactory(type_id='ietf', date=date_today()) group = GroupFactory(acronym='mars') - url = reverse('ietf.secr.sreq.views.no_session',kwargs={'acronym':group.acronym}) + url = reverse('ietf.meeting.views_session_request.no_session', kwargs={'acronym': group.acronym}) self.client.login(username="secretary", password="secretary+password") empty_outbox() - r = self.client.get(url,follow=True) + r = self.client.get(url, follow=True) # If the view invoked by that get throws an exception (such as an integrity error), # the traceback from this test will talk about a TransactionManagementError and # yell about executing queries before the end of an 'atomic' block @@ -932,14 +903,15 @@ def test_not_meeting(self): self.assertEqual(r.status_code, 200) self.assertContains(r, 'A message was sent to notify not having a session') - r = self.client.get(url,follow=True) + r = self.client.get(url, follow=True) self.assertEqual(r.status_code, 200) self.assertContains(r, 'is already marked as not meeting') - self.assertEqual(len(outbox),1) + self.assertEqual(len(outbox), 1) self.assertTrue('Not having a session' in outbox[0]['Subject']) self.assertTrue('session-request@' in outbox[0]['To']) + class RetrievePreviousCase(TestCase): pass @@ -949,7 +921,7 @@ class RetrievePreviousCase(TestCase): # test access by unauthorized -class SessionFormTest(TestCase): +class SessionRequestFormTest(TestCase): def setUp(self): super().setUp() self.meeting = MeetingFactory(type_id='ietf') @@ -1014,19 +986,19 @@ def setUp(self): 'session_set-2-comments': '', 'session_set-2-DELETE': '', } - + def test_valid(self): # Test with three sessions - form = SessionForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting) + form = SessionRequestForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting) self.assertTrue(form.is_valid()) - + # Test with two sessions self.valid_form_data.update({ 'third_session': '', 'session_set-TOTAL_FORMS': '2', 'joint_for_session': '2' }) - form = SessionForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting) + form = SessionRequestForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting) self.assertTrue(form.is_valid()) # Test with one session @@ -1036,9 +1008,9 @@ def test_valid(self): 'joint_for_session': '1', 'session_time_relation': '', }) - form = SessionForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting) + form = SessionRequestForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting) self.assertTrue(form.is_valid()) - + def test_invalid_groups(self): new_form_data = { 'constraint_chair_conflict': 'doesnotexist', @@ -1057,7 +1029,7 @@ def test_valid_group_appears_in_multiple_conflicts(self): 'constraint_tech_overlap': self.group2.acronym, } self.valid_form_data.update(new_form_data) - form = SessionForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting) + form = SessionRequestForm(data=self.valid_form_data, group=self.group1, meeting=self.meeting) self.assertTrue(form.is_valid()) def test_invalid_group_appears_in_multiple_conflicts(self): @@ -1116,7 +1088,7 @@ def test_invalid_joint_for_session(self): 'joint_for_session': [ 'Session 2 can not be the joint session, the session has not been requested.'] }) - + def test_invalid_missing_session_length(self): form = self._invalid_test_helper({ 'session_set-TOTAL_FORMS': '2', @@ -1156,6 +1128,6 @@ def test_invalid_missing_session_length(self): def _invalid_test_helper(self, new_form_data): form_data = dict(self.valid_form_data, **new_form_data) - form = SessionForm(data=form_data, group=self.group1, meeting=self.meeting) + form = SessionRequestForm(data=form_data, group=self.group1, meeting=self.meeting) self.assertFalse(form.is_valid()) return form diff --git a/ietf/meeting/tests_tasks.py b/ietf/meeting/tests_tasks.py index 0c442c4bf7..2c5120a39d 100644 --- a/ietf/meeting/tests_tasks.py +++ b/ietf/meeting/tests_tasks.py @@ -5,25 +5,65 @@ from ietf.utils.test_utils import TestCase from ietf.utils.timezone import date_today from .factories import MeetingFactory -from .tasks import proceedings_content_refresh_task, agenda_data_refresh +from .tasks import ( + proceedings_content_refresh_task, + agenda_data_refresh_task, + agenda_data_refresh_all_task, +) from .tasks import fetch_meeting_attendance_task class TaskTests(TestCase): @patch("ietf.meeting.tasks.generate_agenda_data") - def test_agenda_data_refresh(self, mock_generate): - agenda_data_refresh() + def test_agenda_data_refresh_task(self, mock_generate): + agenda_data_refresh_task() self.assertTrue(mock_generate.called) - self.assertEqual(mock_generate.call_args, call(force_refresh=True)) + self.assertEqual(mock_generate.call_args, call(None, force_refresh=True)) + + mock_generate.reset_mock() + mock_generate.side_effect = RuntimeError + try: + agenda_data_refresh_task() + except Exception as err: + self.fail( + f"agenda_data_refresh_task should not raise exceptions (got {repr(err)})" + ) + + @patch("ietf.meeting.tasks.agenda_data_refresh_task") + @patch("ietf.meeting.tasks.chain") + def test_agenda_data_refresh_all_task(self, mock_chain, mock_agenda_data_refresh): + # Patch the agenda_data_refresh_task task with a mock whose `.map` attribute + # converts its argument, which is expected to be an iterator, to a list + # and returns it. We'll use this to check that the expected task chain + # was set up, but we don't actually run any celery tasks. + mock_agenda_data_refresh.map.side_effect = lambda x: list(x) + + meetings = MeetingFactory.create_batch(5, type_id="ietf") + numbers = sorted(int(m.number) for m in meetings) + agenda_data_refresh_all_task(batch_size=2) + self.assertTrue(mock_chain.called) + # The lists in the call() below are the output of the lambda we patched in + # via mock_agenda_data_refresh.map.side_effect above. I.e., this tests that + # map() was called with the correct batched data. + self.assertEqual( + mock_chain.call_args, + call( + [numbers[0], numbers[1]], + [numbers[2], numbers[3]], + [numbers[4]], + ), + ) + self.assertEqual(mock_agenda_data_refresh.call_count, 0) + self.assertEqual(mock_agenda_data_refresh.map.call_count, 3) @patch("ietf.meeting.tasks.generate_proceedings_content") def test_proceedings_content_refresh_task(self, mock_generate): # Generate a couple of meetings meeting120 = MeetingFactory(type_id="ietf", number="120") # 24 * 5 meeting127 = MeetingFactory(type_id="ietf", number="127") # 24 * 5 + 7 - + # Times to be returned - now_utc = datetime.datetime.now(tz=datetime.timezone.utc) + now_utc = datetime.datetime.now(tz=datetime.UTC) hour_00_utc = now_utc.replace(hour=0) hour_01_utc = now_utc.replace(hour=1) hour_07_utc = now_utc.replace(hour=7) @@ -34,19 +74,19 @@ def test_proceedings_content_refresh_task(self, mock_generate): self.assertEqual(mock_generate.call_count, 1) self.assertEqual(mock_generate.call_args, call(meeting120, force_refresh=True)) mock_generate.reset_mock() - + # hour 01 - should call no meetings with patch("ietf.meeting.tasks.timezone.now", return_value=hour_01_utc): proceedings_content_refresh_task() self.assertEqual(mock_generate.call_count, 0) - + # hour 07 - should call meeting with number % 24 == 0 with patch("ietf.meeting.tasks.timezone.now", return_value=hour_07_utc): proceedings_content_refresh_task() self.assertEqual(mock_generate.call_count, 1) self.assertEqual(mock_generate.call_args, call(meeting127, force_refresh=True)) mock_generate.reset_mock() - + # With all=True, all should be called regardless of time. Reuse hour_01_utc which called none before with patch("ietf.meeting.tasks.timezone.now", return_value=hour_01_utc): proceedings_content_refresh_task(all=True) @@ -61,10 +101,10 @@ def test_fetch_meeting_attendance_task(self, mock_fetch_attendance): MeetingFactory(type_id="ietf", date=today - datetime.timedelta(days=3)), ] data = { - 'created': 1, - 'updated': 2, - 'deleted': 0, - 'processed': 3, + "created": 1, + "updated": 2, + "deleted": 0, + "processed": 3, } mock_fetch_attendance.return_value = [data, data] diff --git a/ietf/meeting/tests_views.py b/ietf/meeting/tests_views.py index f382772485..168999d0aa 100644 --- a/ietf/meeting/tests_views.py +++ b/ietf/meeting/tests_views.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2009-2024, All Rights Reserved +# Copyright The IETF Trust 2009-2025, All Rights Reserved # -*- coding: utf-8 -*- import datetime import io @@ -49,15 +49,18 @@ from ietf.meeting.helpers import send_interim_minutes_reminder, populate_important_dates, update_important_dates from ietf.meeting.models import Session, TimeSlot, Meeting, SchedTimeSessAssignment, Schedule, SessionPresentation, SlideSubmission, SchedulingEvent, Room, Constraint, ConstraintName from ietf.meeting.test_data import make_meeting_test_data, make_interim_meeting, make_interim_test_data -from ietf.meeting.utils import condition_slide_order, generate_proceedings_content +from ietf.meeting.utils import ( + condition_slide_order, + generate_proceedings_content, + diff_meeting_schedules, +) from ietf.meeting.utils import add_event_info_to_session_qs, participants_for_meeting from ietf.meeting.utils import create_recording, delete_recording, get_next_sequence, bluesheet_data from ietf.meeting.views import session_draft_list, parse_agenda_filter_params, sessions_post_save, agenda_extract_schedule from ietf.meeting.views import get_summary_by_area, get_summary_by_type, get_summary_by_purpose, generate_agenda_data from ietf.name.models import SessionStatusName, ImportantDateName, RoleName, ProceedingsMaterialTypeName -from ietf.utils.decorators import skip_coverage from ietf.utils.mail import outbox, empty_outbox, get_payload_text -from ietf.utils.test_runner import TestBlobstoreManager +from ietf.utils.test_runner import TestBlobstoreManager, disable_coverage from ietf.utils.test_utils import TestCase, login_testing_unauthorized, unicontent from ietf.utils.timezone import date_today, time_now @@ -321,11 +324,11 @@ def test_meeting_agenda(self): self.assertContains(r, session.group.parent.acronym.upper()) self.assertContains(r, slot.location.name) self.assertContains(r, "{}-{}".format( - slot.time.astimezone(datetime.timezone.utc).strftime("%H%M"), - (slot.time + slot.duration).astimezone(datetime.timezone.utc).strftime("%H%M"), + slot.time.astimezone(datetime.UTC).strftime("%H%M"), + (slot.time + slot.duration).astimezone(datetime.UTC).strftime("%H%M"), )) self.assertContains(r, "shown in UTC") - updated = meeting.updated().astimezone(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S %Z") + updated = meeting.updated().astimezone(datetime.UTC).strftime("%Y-%m-%d %H:%M:%S %Z") self.assertContains(r, f"Updated {updated}") # text, invalid updated (none) @@ -369,8 +372,8 @@ def test_meeting_agenda(self): self.assertContains(r, session.group.parent.acronym.upper()) self.assertContains(r, slot.location.name) self.assertContains(r, registration_text) - start_time = slot.time.astimezone(datetime.timezone.utc) - end_time = slot.end_time().astimezone(datetime.timezone.utc) + start_time = slot.time.astimezone(datetime.UTC) + end_time = slot.end_time().astimezone(datetime.UTC) self.assertContains(r, '"{}","{}","{}"'.format( start_time.strftime("%Y-%m-%d"), start_time.strftime("%H%M"), @@ -1037,7 +1040,7 @@ def test_important_dates_ical(self): updated = meeting.updated() self.assertIsNotNone(updated) - expected_updated = updated.astimezone(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + expected_updated = updated.astimezone(datetime.UTC).strftime("%Y%m%dT%H%M%SZ") self.assertContains(r, f"DTSTAMP:{expected_updated}") dtstamps_count = r.content.decode("utf-8").count(f"DTSTAMP:{expected_updated}") self.assertEqual(dtstamps_count, meeting.importantdate_set.count()) @@ -1181,8 +1184,8 @@ def test_session_draft_tarfile(self): os.unlink(filename) @skipIf(skip_pdf_tests, skip_message) - @skip_coverage - def test_session_draft_pdf(self): + @disable_coverage() + def test_session_draft_pdf(self): # pragma: no cover session, filenames = self.build_session_setup() try: url = urlreverse('ietf.meeting.views.session_draft_pdf', kwargs={'num':session.meeting.number,'acronym':session.group.acronym}) @@ -2117,8 +2120,8 @@ def test_editor_time_zone(self): # strftime() does not seem to support hours without leading 0, so do this manually time_label_string = f'{ts_start.hour:d}:{ts_start.minute:02d} - {ts_end.hour:d}:{ts_end.minute:02d}' self.assertIn(time_label_string, time_label.text()) - self.assertEqual(time_label.attr('data-start'), ts_start.astimezone(datetime.timezone.utc).isoformat()) - self.assertEqual(time_label.attr('data-end'), ts_end.astimezone(datetime.timezone.utc).isoformat()) + self.assertEqual(time_label.attr('data-start'), ts_start.astimezone(datetime.UTC).isoformat()) + self.assertEqual(time_label.attr('data-end'), ts_end.astimezone(datetime.UTC).isoformat()) ts_swap = time_label.find('.swap-timeslot-col') origin_label = ts_swap.attr('data-origin-label') @@ -2129,8 +2132,8 @@ def test_editor_time_zone(self): timeslot_elt = pq(f'#timeslot{timeslot.pk}') self.assertEqual(len(timeslot_elt), 1) - self.assertEqual(timeslot_elt.attr('data-start'), ts_start.astimezone(datetime.timezone.utc).isoformat()) - self.assertEqual(timeslot_elt.attr('data-end'), ts_end.astimezone(datetime.timezone.utc).isoformat()) + self.assertEqual(timeslot_elt.attr('data-start'), ts_start.astimezone(datetime.UTC).isoformat()) + self.assertEqual(timeslot_elt.attr('data-end'), ts_end.astimezone(datetime.UTC).isoformat()) timeslot_label = pq(f'#timeslot{timeslot.pk} .time-label') self.assertEqual(len(timeslot_label), 1) @@ -4751,7 +4754,7 @@ def _approval_url(slidesub): 0, "second session proposed slides should be linked for approval", ) - + class EditScheduleListTests(TestCase): def setUp(self): @@ -4766,73 +4769,151 @@ def test_list_schedules(self): self.assertTrue(r.status_code, 200) def test_diff_schedules(self): - meeting = make_meeting_test_data() - - url = urlreverse('ietf.meeting.views.diff_schedules',kwargs={'num':meeting.number}) - login_testing_unauthorized(self,"secretary", url) - r = self.client.get(url) - self.assertTrue(r.status_code, 200) - - from_schedule = Schedule.objects.get(meeting=meeting, name="test-unofficial-schedule") - - session1 = Session.objects.filter(meeting=meeting, group__acronym='mars').first() - session2 = Session.objects.filter(meeting=meeting, group__acronym='ames').first() - session3 = SessionFactory(meeting=meeting, group=Group.objects.get(acronym='mars'), - attendees=10, requested_duration=datetime.timedelta(minutes=70), - add_to_schedule=False) - SchedulingEvent.objects.create(session=session3, status_id='schedw', by=Person.objects.first()) - - slot2 = TimeSlot.objects.filter(meeting=meeting, type='regular').order_by('-time').first() - slot3 = TimeSlot.objects.create( - meeting=meeting, type_id='regular', location=slot2.location, - duration=datetime.timedelta(minutes=60), - time=slot2.time + datetime.timedelta(minutes=60), + # Create meeting and some time slots + meeting = MeetingFactory(type_id="ietf", populate_schedule=False) + rooms = RoomFactory.create_batch(2, meeting=meeting) + # first index is room, second is time + timeslots = [ + [ + TimeSlotFactory( + location=room, + meeting=meeting, + time=datetime.datetime.combine( + meeting.date, datetime.time(9, 0, tzinfo=datetime.UTC) + ) + ), + TimeSlotFactory( + location=room, + meeting=meeting, + time=datetime.datetime.combine( + meeting.date, datetime.time(10, 0, tzinfo=datetime.UTC) + ) + ), + TimeSlotFactory( + location=room, + meeting=meeting, + time=datetime.datetime.combine( + meeting.date, datetime.time(11, 0, tzinfo=datetime.UTC) + ) + ), + ] + for room in rooms + ] + sessions = SessionFactory.create_batch( + 5, meeting=meeting, add_to_schedule=False ) - # copy - new_url = urlreverse("ietf.meeting.views.new_meeting_schedule", kwargs=dict(num=meeting.number, owner=from_schedule.owner_email(), name=from_schedule.name)) - r = self.client.post(new_url, { - 'name': "newtest", - 'public': "on", - }) - self.assertNoFormPostErrors(r) + from_schedule = ScheduleFactory(meeting=meeting) + to_schedule = ScheduleFactory(meeting=meeting) - to_schedule = Schedule.objects.get(meeting=meeting, name='newtest') + # sessions[0]: not scheduled in from_schedule, scheduled in to_schedule + SchedTimeSessAssignment.objects.create( + schedule=to_schedule, + session=sessions[0], + timeslot=timeslots[0][0], + ) + # sessions[1]: scheduled in from_schedule, not scheduled in to_schedule + SchedTimeSessAssignment.objects.create( + schedule=from_schedule, + session=sessions[1], + timeslot=timeslots[0][0], + ) + # sessions[2]: moves rooms, not time + SchedTimeSessAssignment.objects.create( + schedule=from_schedule, + session=sessions[2], + timeslot=timeslots[0][1], + ) + SchedTimeSessAssignment.objects.create( + schedule=to_schedule, + session=sessions[2], + timeslot=timeslots[1][1], + ) + # sessions[3]: moves time, not room + SchedTimeSessAssignment.objects.create( + schedule=from_schedule, + session=sessions[3], + timeslot=timeslots[1][1], + ) + SchedTimeSessAssignment.objects.create( + schedule=to_schedule, + session=sessions[3], + timeslot=timeslots[1][2], + ) + # sessions[4]: moves room and time + SchedTimeSessAssignment.objects.create( + schedule=from_schedule, + session=sessions[4], + timeslot=timeslots[1][0], + ) + SchedTimeSessAssignment.objects.create( + schedule=to_schedule, + session=sessions[4], + timeslot=timeslots[0][2], + ) - # make some changes + # Check the raw diffs + raw_diffs = diff_meeting_schedules(from_schedule, to_schedule) + self.assertCountEqual( + raw_diffs, + [ + { + "change": "schedule", + "session": sessions[0].pk, + "to": timeslots[0][0].pk, + }, + { + "change": "unschedule", + "session": sessions[1].pk, + "from": timeslots[0][0].pk, + }, + { + "change": "move", + "session": sessions[2].pk, + "from": timeslots[0][1].pk, + "to": timeslots[1][1].pk, + }, + { + "change": "move", + "session": sessions[3].pk, + "from": timeslots[1][1].pk, + "to": timeslots[1][2].pk, + }, + { + "change": "move", + "session": sessions[4].pk, + "from": timeslots[1][0].pk, + "to": timeslots[0][2].pk, + }, + ] + ) - edit_url = urlreverse("ietf.meeting.views.edit_meeting_schedule", kwargs=dict(num=meeting.number, owner=to_schedule.owner_email(), name=to_schedule.name)) + # Check the view + url = urlreverse("ietf.meeting.views.diff_schedules", + kwargs={"num": meeting.number}) + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + self.assertTrue(r.status_code, 200) - # schedule session - r = self.client.post(edit_url, { - 'action': 'assign', - 'timeslot': slot3.pk, - 'session': session3.pk, - }) - self.assertEqual(json.loads(r.content)['success'], True) - # unschedule session - r = self.client.post(edit_url, { - 'action': 'unassign', - 'session': session1.pk, - }) - self.assertEqual(json.loads(r.content)['success'], True) - # move session - r = self.client.post(edit_url, { - 'action': 'assign', - 'timeslot': slot2.pk, - 'session': session2.pk, + # with show room changes disabled - does not show sessions[2] because it did + # not change time + r = self.client.get(url, { + "from_schedule": from_schedule.name, + "to_schedule": to_schedule.name, }) - self.assertEqual(json.loads(r.content)['success'], True) + self.assertTrue(r.status_code, 200) + q = PyQuery(r.content) + self.assertEqual(len(q(".schedule-diffs tr")), 4 + 1) - # now get differences + # with show room changes enabled - shows all changes r = self.client.get(url, { - 'from_schedule': from_schedule.name, - 'to_schedule': to_schedule.name, + "from_schedule": from_schedule.name, + "to_schedule": to_schedule.name, + "show_room_changes": "on", }) self.assertTrue(r.status_code, 200) - q = PyQuery(r.content) - self.assertEqual(len(q(".schedule-diffs tr")), 3+1) + self.assertEqual(len(q(".schedule-diffs tr")), 5 + 1) def test_delete_schedule(self): url = urlreverse('ietf.meeting.views.delete_schedule', @@ -5221,7 +5302,9 @@ def test_upcoming_ical(self): assert_ical_response_is_valid(self, r, expected_event_summaries=expected_event_summaries, expected_event_count=len(expected_event_summaries)) - self.assertContains(r, 'Remote instructions: https://someurl.example.com') + # Unfold long lines that might have been folded by iCal + content_unfolded = r.content.decode('utf-8').replace('\r\n ', '') + self.assertIn('Remote instructions: https://someurl.example.com', content_unfolded) Session.objects.filter(meeting__type_id='interim').update(remote_instructions='') r = self.client.get(url) @@ -5229,11 +5312,12 @@ def test_upcoming_ical(self): assert_ical_response_is_valid(self, r, expected_event_summaries=expected_event_summaries, expected_event_count=len(expected_event_summaries)) - self.assertNotContains(r, 'Remote instructions:') + content_unfolded = r.content.decode('utf-8').replace('\r\n ', '') + self.assertNotIn('Remote instructions:', content_unfolded) updated = meeting.updated() self.assertIsNotNone(updated) - expected_updated = updated.astimezone(datetime.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + expected_updated = updated.astimezone(datetime.UTC).strftime("%Y%m%dT%H%M%SZ") self.assertContains(r, f"DTSTAMP:{expected_updated}") # With default cached_updated, 1970-01-01 @@ -7067,7 +7151,7 @@ def test_disapprove_proposed_slides(self): self.assertFalse(exists_in_storage("staging", submission.filename)) r = self.client.get(url) self.assertEqual(r.status_code, 200) - self.assertRegex(r.content.decode(), r"These\s+slides\s+have\s+already\s+been\s+rejected") + self.assertRegex(r.content.decode(), r"These\s+slides\s+have\s+already\s+been\s+declined") @override_settings(MEETECHO_API_CONFIG="fake settings") # enough to trigger API calls @patch("ietf.meeting.views.SlidesManager") @@ -7261,6 +7345,67 @@ def test_submit_and_approve_multiple_versions(self, mock_slides_manager_cls): fd.close() self.assertIn('third version', contents) + @override_settings( + MEETECHO_API_CONFIG="fake settings" + ) # enough to trigger API calls + @patch("ietf.meeting.views.SlidesManager") + def test_notify_meetecho_of_all_slides(self, mock_slides_manager_cls): + for meeting_type in ["ietf", "interim"]: + # Reset for the sake of the second iteration + self.client.logout() + mock_slides_manager_cls.reset_mock() + + session = SessionFactory(meeting__type_id=meeting_type) + meeting = session.meeting + + # bad meeting + url = urlreverse( + "ietf.meeting.views.notify_meetecho_of_all_slides", + kwargs={"num": 9999, "acronym": session.group.acronym}, + ) + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + self.assertEqual(r.status_code, 404) + r = self.client.post(url) + self.assertEqual(r.status_code, 404) + self.assertFalse(mock_slides_manager_cls.called) + self.client.logout() + + # good meeting + url = urlreverse( + "ietf.meeting.views.notify_meetecho_of_all_slides", + kwargs={"num": meeting.number, "acronym": session.group.acronym}, + ) + login_testing_unauthorized(self, "secretary", url) + r = self.client.get(url) + self.assertEqual(r.status_code, 405) + self.assertFalse(mock_slides_manager_cls.called) + mock_slides_manager = mock_slides_manager_cls.return_value + mock_slides_manager.send_update.return_value = True + r = self.client.post(url) + self.assertEqual(r.status_code, 302) + self.assertEqual(mock_slides_manager.send_update.call_count, 1) + self.assertEqual(mock_slides_manager.send_update.call_args, call(session)) + r = self.client.get(r["Location"]) + messages = list(r.context["messages"]) + self.assertEqual(len(messages), 1) + self.assertEqual( + str(messages[0]), f"Notified Meetecho about slides for {session}" + ) + + mock_slides_manager.send_update.reset_mock() + mock_slides_manager.send_update.return_value = False + r = self.client.post(url) + self.assertEqual(r.status_code, 302) + self.assertEqual(mock_slides_manager.send_update.call_count, 1) + self.assertEqual(mock_slides_manager.send_update.call_args, call(session)) + r = self.client.get(r["Location"]) + messages = list(r.context["messages"]) + self.assertEqual(len(messages), 1) + self.assertIn( + "No sessions were eligible for Meetecho slides update.", str(messages[0]) + ) + @override_settings(IETF_NOTES_URL='https://notes.ietf.org/') class ImportNotesTests(TestCase): @@ -7555,7 +7700,7 @@ def test_meeting_requests(self): ) def _sreq_edit_link(sess): return urlreverse( - 'ietf.secr.sreq.views.edit', + 'ietf.meeting.views_session_request.edit_request', kwargs={ 'num': meeting.number, 'acronym': sess.group.acronym, diff --git a/ietf/meeting/urls.py b/ietf/meeting/urls.py index 18b123b4d8..a038e1cfe6 100644 --- a/ietf/meeting/urls.py +++ b/ietf/meeting/urls.py @@ -1,10 +1,10 @@ -# Copyright The IETF Trust 2007-2024, All Rights Reserved +# Copyright The IETF Trust 2007-2025, All Rights Reserved from django.conf import settings from django.urls import include from django.views.generic import RedirectView -from ietf.meeting import views, views_proceedings +from ietf.meeting import views, views_proceedings, views_session_request from ietf.utils.urls import url class AgendaRedirectView(RedirectView): @@ -15,6 +15,7 @@ def get_redirect_url(self, *args, **kwargs): safe_for_all_meeting_types = [ url(r'^session/(?P[-a-z0-9]+)/?$', views.session_details), + url(r'^session/(?P[-a-z0-9]+)/send_slide_notifications$', views.notify_meetecho_of_all_slides), url(r'^session/(?P\d+)/drafts$', views.add_session_drafts), url(r'^session/(?P\d+)/recordings$', views.add_session_recordings), url(r'^session/(?P\d+)/attendance$', views.session_attendance), @@ -30,7 +31,7 @@ def get_redirect_url(self, *args, **kwargs): url(r'^session/(?P\d+)/doc/%(name)s/remove$' % settings.URL_REGEXPS, views.remove_sessionpresentation), url(r'^session/(?P\d+)\.ics$', views.agenda_ical), url(r'^sessions/(?P[-a-z0-9]+)\.ics$', views.agenda_ical), - url(r'^slidesubmission/(?P\d+)$', views.approve_proposed_slides) + url(r'^slidesubmission/(?P\d+)$', views.approve_proposed_slides), ] @@ -108,6 +109,8 @@ def get_redirect_url(self, *args, **kwargs): url(r'^important-dates.(?Pics)$', views.important_dates), url(r'^proceedings/meetinghosts/edit/', views_proceedings.edit_meetinghosts), url(r'^proceedings/meetinghosts/(?P\d+)/logo/$', views_proceedings.meetinghost_logo), + url(r'^session/request/%(acronym)s/edit/$' % settings.URL_REGEXPS, views_session_request.edit_request), + url(r'^session/request/%(acronym)s/view/$' % settings.URL_REGEXPS, views_session_request.view_request), ] urlpatterns = [ @@ -127,6 +130,13 @@ def get_redirect_url(self, *args, **kwargs): url(r'^upcoming/?$', views.upcoming), url(r'^upcoming\.ics/?$', views.upcoming_ical), url(r'^upcoming\.json/?$', views.upcoming_json), + url(r'^session/request/$', views_session_request.list_view), + url(r'^session/request/%(acronym)s/new/$' % settings.URL_REGEXPS, views_session_request.new_request), + url(r'^session/request/%(acronym)s/approve/$' % settings.URL_REGEXPS, views_session_request.approve_request), + url(r'^session/request/%(acronym)s/no_session/$' % settings.URL_REGEXPS, views_session_request.no_session), + url(r'^session/request/%(acronym)s/cancel/$' % settings.URL_REGEXPS, views_session_request.cancel_request), + url(r'^session/request/%(acronym)s/confirm/$' % settings.URL_REGEXPS, views_session_request.confirm), + url(r'^session/request/status/$', views_session_request.status), url(r'^session/(?P\d+)/agenda_materials$', views.session_materials), url(r'^session/(?P\d+)/cancel/?', views.cancel_session), url(r'^session/(?P\d+)/edit/?', views.edit_session), @@ -140,4 +150,3 @@ def get_redirect_url(self, *args, **kwargs): url(r'^(?P\d+)/', include(safe_for_all_meeting_types)), url(r'^(?Pinterim-[a-z0-9-]+)/', include(safe_for_all_meeting_types)), ] - diff --git a/ietf/meeting/utils.py b/ietf/meeting/utils.py index f6925269aa..bdf3d3d3d3 100644 --- a/ietf/meeting/utils.py +++ b/ietf/meeting/utils.py @@ -2,10 +2,12 @@ # -*- coding: utf-8 -*- import datetime import itertools +from contextlib import suppress +from dataclasses import dataclass + import jsonschema import os import requests -from hashlib import sha384 import pytz import subprocess @@ -27,16 +29,33 @@ import debug # pyflakes:ignore from ietf.dbtemplate.models import DBTemplate -from ietf.doc.storage_utils import store_bytes, store_str -from ietf.meeting.models import (Session, SchedulingEvent, TimeSlot, - Constraint, SchedTimeSessAssignment, SessionPresentation, Attended, - Registration, Meeting, RegistrationTicket) -from ietf.doc.models import Document, State, NewRevisionDocEvent, StateDocEvent +from ietf.doc.storage_utils import store_bytes, store_str, AlreadyExistsError +from ietf.meeting.models import ( + Session, + SchedulingEvent, + TimeSlot, + Constraint, + SchedTimeSessAssignment, + SessionPresentation, + Attended, + Registration, + Meeting, + RegistrationTicket, +) +from ietf.blobdb.models import ResolvedMaterial +from ietf.doc.models import ( + Document, + State, + NewRevisionDocEvent, + StateDocEvent, + StoredObject, +) from ietf.doc.models import DocEvent from ietf.group.models import Group from ietf.group.utils import can_manage_materials from ietf.name.models import SessionStatusName, ConstraintName, DocTypeName from ietf.person.models import Person +from ietf.utils import markdown from ietf.utils.html import clean_html from ietf.utils.log import log from ietf.utils.timezone import date_today @@ -221,6 +240,7 @@ def save_bluesheet(request, session, file, encoding='utf-8'): save_error = handle_upload_file(file, filename, session.meeting, 'bluesheets', request=request, encoding=encoding) if not save_error: doc.save_with_history([e]) + resolve_uploaded_material(meeting=session.meeting, doc=doc) return save_error @@ -833,6 +853,330 @@ def write_doc_for_session(session, type_id, filename, contents): store_str(type_id, filename.name, contents) return None + +@dataclass +class BlobSpec: + bucket: str + name: str + + +def resolve_one_material( + doc: Document, rev: str | None, ext: str | None +) -> BlobSpec | None: + if doc.type_id is None: + log(f"Cannot resolve a doc with no type: {doc.name}") + return None + + # Get the Document's base name. It may or may not have an extension. + if rev is None: + basename = Path(doc.get_base_name()) + else: + basename = Path(f"{doc.name}-{int(rev):02d}") + + # If the document's file exists, the blob is _always_ named with this stem, + # even if it's different from the original. + blob_stem = Path(f"{doc.name}-{rev or doc.rev}") + + # If we have an extension, either from the URL or the Document's base name, look up + # the blob or file or return 404. N.b. the suffix check needs adjustment to handle + # a bare "." extension when we reach py3.14. + if ext or basename.suffix != "": + if ext: + blob_name = str(blob_stem.with_suffix(ext)) + else: + blob_name = str(blob_stem.with_suffix(basename.suffix)) + + # See if we have a stored object under that name + preferred_blob = ( + StoredObject.objects.exclude_deleted() + .filter(store=doc.type_id, name=blob_name) + .first() + ) + if preferred_blob is not None: + return BlobSpec( + bucket=preferred_blob.store, + name=preferred_blob.name, + ) + # No stored object, fall back to the file system. + filename = Path(doc.get_file_path()) / basename # use basename for file + if filename.is_file(): + return BlobSpec( + bucket=doc.type_id, + name=str(blob_stem.with_suffix(filename.suffix)), + ) + else: + return None + + # No extension has been specified so far, so look one up. + matching_stored_objects = ( + StoredObject.objects.exclude_deleted() + .filter( + store=doc.type_id, + name__startswith=f"{blob_stem}.", # anchor to end with trailing "." + ) + .order_by("name") + ) # orders by suffix + blob_ext_choices = { + Path(stored_obj.name).suffix: stored_obj + for stored_obj in matching_stored_objects + } + + # Short-circuit to return pdf if present + if ".pdf" in blob_ext_choices: + pdf_blob = blob_ext_choices[".pdf"] + return BlobSpec( + bucket=pdf_blob.store, + name=str(blob_stem.with_suffix(".pdf")), + ) + + # Now look for files + filename = Path(doc.get_file_path()) / basename + file_ext_choices = { + # Construct a map from suffix to full filename + fn.suffix: fn.name + for fn in sorted(filename.parent.glob(filename.stem + ".*")) + } + + # Short-circuit to return pdf if we have the file + if ".pdf" in file_ext_choices: + return BlobSpec( + bucket=doc.type_id, + name=str(blob_stem.with_suffix(".pdf")), + ) + + all_exts = set(blob_ext_choices.keys()).union(file_ext_choices.keys()) + if len(all_exts) > 0: + preferred_ext = sorted(all_exts)[0] + if preferred_ext in blob_ext_choices: + preferred_blob = blob_ext_choices[preferred_ext] + return BlobSpec( + bucket=preferred_blob.store, + name=preferred_blob.name, + ) + else: + return BlobSpec( + bucket=doc.type_id, + name=str(blob_stem.with_suffix(preferred_ext)), + ) + + return None + + +def resolve_materials_for_one_meeting(meeting: Meeting): + start_time = timezone.now() + meeting_documents = ( + Document.objects.filter( + type_id__in=settings.MATERIALS_TYPES_SERVED_BY_WORKER + ).filter( + Q(session__meeting=meeting) | Q(proceedingsmaterial__meeting=meeting) + ) + ).distinct() + + resolved = [] + for doc in meeting_documents: + # request by doc name with no rev + blob = resolve_one_material(doc, rev=None, ext=None) + if blob is not None: + resolved.append( + ResolvedMaterial( + name=doc.name, + meeting_number=meeting.number, + bucket=blob.bucket, + blob=blob.name, + ) + ) + # request by doc name + rev + blob = resolve_one_material(doc, rev=doc.rev, ext=None) + if blob is not None: + resolved.append( + ResolvedMaterial( + name=f"{doc.name}-{doc.rev:02}", + meeting_number=meeting.number, + bucket=blob.bucket, + blob=blob.name, + ) + ) + # for other revisions, only need request by doc name + rev + other_revisions = doc.revisions_by_newrevisionevent() + other_revisions.remove(doc.rev) + for rev in other_revisions: + blob = resolve_one_material(doc, rev=rev, ext=None) + if blob is not None: + resolved.append( + ResolvedMaterial( + name=f"{doc.name}-{rev:02}", + meeting_number=meeting.number, + bucket=blob.bucket, + blob=blob.name, + ) + ) + ResolvedMaterial.objects.bulk_create( + resolved, + update_conflicts=True, + unique_fields=["name", "meeting_number"], + update_fields=["bucket", "blob"], + ) + # Warn if any files were updated during the above process + last_update = meeting_documents.aggregate(Max("time"))["time__max"] + if last_update and last_update > start_time: + log( + f"Warning: materials for meeting {meeting.number} " + "changed during ResolvedMaterial update" + ) + +def resolve_uploaded_material(meeting: Meeting, doc: Document): + resolved = [] + blob = resolve_one_material(doc, rev=None, ext=None) + if blob is not None: + resolved.append( + ResolvedMaterial( + name=doc.name, + meeting_number=meeting.number, + bucket=blob.bucket, + blob=blob.name, + ) + ) + # request by doc name + rev + blob = resolve_one_material(doc, rev=doc.rev, ext=None) + if blob is not None: + resolved.append( + ResolvedMaterial( + name=f"{doc.name}-{doc.rev:02}", + meeting_number=meeting.number, + bucket=blob.bucket, + blob=blob.name, + ) + ) + ResolvedMaterial.objects.bulk_create( + resolved, + update_conflicts=True, + unique_fields=["name", "meeting_number"], + update_fields=["bucket", "blob"], + ) + + +def store_blob_for_one_material_file(doc: Document, rev: str, filepath: Path): + if not settings.ENABLE_BLOBSTORAGE: + raise RuntimeError("Cannot store blobs: ENABLE_BLOBSTORAGE is False") + + bucket = doc.type_id + if bucket not in settings.MATERIALS_TYPES_SERVED_BY_WORKER: + raise ValueError(f"Bucket {bucket} not found for doc {doc.name}.") + blob_stem = f"{doc.name}-{rev}" + suffix = filepath.suffix # includes leading "." + + # Store the file + try: + file_bytes = filepath.read_bytes() + except Exception as err: + log(f"Failed to read {filepath}: {err}") + raise + with suppress(AlreadyExistsError): + store_bytes( + kind=bucket, + name= blob_stem + suffix, + content=file_bytes, + mtime=datetime.datetime.fromtimestamp( + filepath.stat().st_mtime, + tz=datetime.UTC, + ), + allow_overwrite=False, + doc_name=doc.name, + doc_rev=rev, + ) + + # Special case: pre-render markdown into HTML as .md.html + if suffix == ".md": + try: + markdown_source = file_bytes.decode("utf-8") + except UnicodeDecodeError as err: + log(f"Unable to decode {filepath} as UTF-8, treating as latin-1: {err}") + markdown_source = file_bytes.decode("latin-1") + # render the markdown + try: + html = render_to_string( + "minimal.html", + { + "content": markdown.markdown(markdown_source), + "title": blob_stem, + "static_ietf_org": settings.STATIC_IETF_ORG, + }, + ) + except Exception as err: + log(f"Failed to render markdown for {filepath}: {err}") + else: + # Don't overwrite, but don't fail if the blob exists + with suppress(AlreadyExistsError): + store_str( + kind=bucket, + name=blob_stem + ".md.html", + content=html, + allow_overwrite=False, + doc_name=doc.name, + doc_rev=rev, + content_type="text/html;charset=utf-8", + ) + + +def store_blobs_for_one_material_doc(doc: Document): + """Ensure that all files related to a materials Document are in the blob store""" + if doc.type_id not in settings.MATERIALS_TYPES_SERVED_BY_WORKER: + log(f"This method does not handle docs of type {doc.name}") + return + + # Store files for current Document / rev + file_path = Path(doc.get_file_path()) + base_name = Path(doc.get_base_name()) + # .stem would remove directories, so use .with_suffix("") + base_name_stem = str(base_name.with_suffix("")) + if base_name_stem.endswith(".") and base_name.suffix == "": + # In Python 3.14, a trailing "." is a valid suffix, but in prior versions + # it is left as part of the stem. The suffix check ensures that either way, + # only a single "." will be removed. + base_name_stem = base_name_stem[:-1] + # Add any we find without the rev + for file_to_store in file_path.glob(base_name_stem + ".*"): + if not (file_to_store.is_file()): + continue + try: + store_blob_for_one_material_file(doc, doc.rev, file_to_store) + except Exception as err: + log( + f"Failed to store blob for {doc} rev {doc.rev} " + f"from {file_to_store}: {err}" + ) + + # Get other revisions + for rev in doc.revisions_by_newrevisionevent(): + if rev == doc.rev: + continue # already handled this + + # Add some that have the rev + for file_to_store in file_path.glob(doc.name + f"-{rev}.*"): + if not file_to_store.is_file(): + continue + try: + store_blob_for_one_material_file(doc, rev, file_to_store) + except Exception as err: + log( + f"Failed to store blob for {doc} rev {rev} " + f"from {file_to_store}: {err}" + ) + + +def store_blobs_for_one_meeting(meeting: Meeting): + meeting_documents = ( + Document.objects.filter( + type_id__in=settings.MATERIALS_TYPES_SERVED_BY_WORKER + ).filter( + Q(session__meeting=meeting) | Q(proceedingsmaterial__meeting=meeting) + ) + ).distinct() + + for doc in meeting_documents: + store_blobs_for_one_material_doc(doc) + + def create_recording(session, url, title=None, user=None): ''' Creates the Document type=recording, setting external_url and creating @@ -1026,12 +1370,44 @@ def generate_proceedings_content(meeting, force_refresh=False): :meeting: meeting whose proceedings should be rendered :force_refresh: true to force regeneration and cache refresh """ - cache = caches["default"] - cache_version = Document.objects.filter(session__meeting__number=meeting.number).aggregate(Max('time'))["time__max"] - # Include proceedings_final in the bare_key so we'll always reflect that accurately, even at the cost of - # a recomputation in the view - bare_key = f"proceedings.{meeting.number}.{cache_version}.final={meeting.proceedings_final}" - cache_key = sha384(bare_key.encode("utf8")).hexdigest() + cache = caches["proceedings"] + key_components = [ + "proceedings", + str(meeting.number), + ] + if meeting.proceedings_final: + # Freeze the cache key once proceedings are finalized. Further changes will + # not be picked up until the cache expires or is refreshed by the + # proceedings_content_refresh_task() + key_components.append("final") + else: + # Build a cache key that changes when materials are modified. For all but drafts, + # use the last modification time of the document. Exclude drafts from this because + # revisions long after the meeting ends will otherwise show up as changes and + # incorrectly invalidate the cache. Instead, include an ordered list of the + # drafts linked to the meeting so adding or removing drafts will trigger a + # recalculation. The list is long but that doesn't matter because we hash it into + # a fixed-length key. + meeting_docs = Document.objects.filter(session__meeting__number=meeting.number) + last_materials_update = ( + meeting_docs.exclude(type_id="draft") + .filter(session__meeting__number=meeting.number) + .aggregate(Max("time"))["time__max"] + ) + draft_names = ( + meeting_docs + .filter(type_id="draft") + .order_by("name") + .values_list("name", flat=True) + ) + key_components += [ + last_materials_update.isoformat() if last_materials_update else "-", + ",".join(draft_names), + ] + + # Key is potentially long, but the "proceedings" cache hashes it to a fixed + # length. If that changes, hash it separately here first. + cache_key = ".".join(key_components) if not force_refresh: cached_content = cache.get(cache_key, None) if cached_content is not None: diff --git a/ietf/meeting/views.py b/ietf/meeting/views.py index 7fa3d21259..731dfad88f 100644 --- a/ietf/meeting/views.py +++ b/ietf/meeting/views.py @@ -9,6 +9,7 @@ import json import math import os + import pytz import re import tarfile @@ -27,17 +28,19 @@ from django import forms from django.core.cache import caches +from django.core.files.storage import storages from django.shortcuts import render, redirect, get_object_or_404 from django.http import (HttpResponse, HttpResponseRedirect, HttpResponseForbidden, HttpResponseNotFound, Http404, HttpResponseBadRequest, - JsonResponse, HttpResponseGone, HttpResponseNotAllowed) + JsonResponse, HttpResponseGone, HttpResponseNotAllowed, + FileResponse) from django.conf import settings from django.contrib import messages from django.contrib.auth.decorators import login_required from django.core.exceptions import ValidationError from django.core.files.uploadedfile import SimpleUploadedFile from django.core.validators import URLValidator -from django.urls import reverse,reverse_lazy +from django.urls import reverse, reverse_lazy, NoReverseMatch from django.db.models import F, Max, Q from django.forms.models import modelform_factory, inlineformset_factory from django.template import TemplateDoesNotExist @@ -48,18 +51,25 @@ from django.views.decorators.cache import cache_page from django.views.decorators.csrf import ensure_csrf_cookie, csrf_exempt from django.views.generic import RedirectView +from rest_framework.status import HTTP_404_NOT_FOUND import debug # pyflakes:ignore from ietf.doc.fields import SearchableDocumentsField from ietf.doc.models import Document, State, DocEvent, NewRevisionDocEvent -from ietf.doc.storage_utils import remove_from_storage, retrieve_bytes, store_file +from ietf.doc.storage_utils import ( + remove_from_storage, + retrieve_bytes, + store_file, +) from ietf.group.models import Group from ietf.group.utils import can_manage_session_materials, can_manage_some_groups, can_manage_group from ietf.person.models import Person, User from ietf.ietfauth.utils import role_required, has_role, user_is_person from ietf.mailtrigger.utils import gather_address_lists -from ietf.meeting.models import Meeting, Session, Schedule, FloorPlan, SessionPresentation, TimeSlot, SlideSubmission, Attended +from ietf.meeting.models import Meeting, Session, Schedule, FloorPlan, \ + SessionPresentation, TimeSlot, SlideSubmission, Attended +from ..blobdb.models import ResolvedMaterial from ietf.meeting.models import ImportantDate, SessionStatusName, SchedulingEvent, SchedTimeSessAssignment, Room, TimeSlotTypeName from ietf.meeting.models import Registration from ietf.meeting.forms import ( CustomDurationField, SwapDaysForm, SwapTimeslotsForm, ImportMinutesForm, @@ -83,7 +93,8 @@ finalize, generate_proceedings_content, organize_proceedings_sessions, - sort_accept_tuple, + resolve_uploaded_material, + sort_accept_tuple, store_blobs_for_one_material_doc, ) from ietf.meeting.utils import add_event_info_to_session_qs from ietf.meeting.utils import session_time_for_sorting @@ -120,6 +131,8 @@ from icalendar import Calendar, Event from ietf.doc.templatetags.ietf_filters import absurl +from ..api.ietf_utils import requires_api_token +from ..blobdb.storage import BlobdbStorage, BlobFile request_summary_exclude_group_types = ['team'] @@ -153,7 +166,7 @@ def materials(request, num=None): begin_date = meeting.get_submission_start_date() cut_off_date = meeting.get_submission_cut_off_date() cor_cut_off_date = meeting.get_submission_correction_date() - today_utc = date_today(datetime.timezone.utc) + today_utc = date_today(datetime.UTC) old = timezone.now() - datetime.timedelta(days=1) if settings.SERVER_MODE != 'production' and '_testoverride' in request.GET: pass @@ -245,21 +258,32 @@ def current_materials(request): raise Http404('No such meeting') -def _get_materials_doc(meeting, name): +def _get_materials_doc(name, meeting=None): """Get meeting materials document named by name - Raises Document.DoesNotExist if a match cannot be found. + Raises Document.DoesNotExist if a match cannot be found. If meeting is None, + matches a name that is associated with _any_ meeting. """ + + def _matches_meeting(doc, meeting=None): + if meeting is None: + return doc.get_related_meeting() is not None + return doc.get_related_meeting() == meeting + # try an exact match first doc = Document.objects.filter(name=name).first() - if doc is not None and doc.get_related_meeting() == meeting: + if doc is not None and _matches_meeting(doc, meeting): return doc, None + # try parsing a rev number if "-" in name: docname, rev = name.rsplit("-", 1) if len(rev) == 2 and rev.isdigit(): doc = Document.objects.get(name=docname) # may raise Document.DoesNotExist - if doc.get_related_meeting() == meeting and rev in doc.revisions_by_newrevisionevent(): + if ( + _matches_meeting(doc, meeting) + and rev in doc.revisions_by_newrevisionevent() + ): return doc, rev # give up raise Document.DoesNotExist @@ -277,7 +301,7 @@ def materials_document(request, document, num=None, ext=None): meeting = get_meeting(num, type_in=["ietf", "interim"]) num = meeting.number try: - doc, rev = _get_materials_doc(meeting=meeting, name=document) + doc, rev = _get_materials_doc(name=document, meeting=meeting) except Document.DoesNotExist: raise Http404("No such document for meeting %s" % num) @@ -305,7 +329,7 @@ def materials_document(request, document, num=None, ext=None): old_proceedings_format = meeting.number.isdigit() and int(meeting.number) <= 96 if settings.MEETING_MATERIALS_SERVE_LOCALLY or old_proceedings_format: bytes = filename.read_bytes() - mtype, chset = get_mime_type(bytes) + mtype, chset = get_mime_type(bytes) # chset does not consider entire file! content_type = "%s; charset=%s" % (mtype, chset) if filename.suffix == ".md" and mtype == "text/plain": @@ -315,14 +339,24 @@ def materials_document(request, document, num=None, ext=None): content_type = content_type.replace("plain", "markdown", 1) break elif atype[0] == "text/html": + # Render markdown, allowing that charset may be inaccurate. + try: + md_src = bytes.decode( + "utf-8" if chset in ["ascii", "us-ascii"] else chset + ) + except UnicodeDecodeError: + # latin-1, aka iso8859-1, accepts all 8-bit code points + md_src = bytes.decode("latin-1") + content = markdown.markdown(md_src) # a string bytes = render_to_string( "minimal.html", { - "content": markdown.markdown(bytes.decode(encoding=chset)), + "content": content, "title": filename.name, + "static_ietf_org": settings.STATIC_IETF_ORG, }, - ) - content_type = content_type.replace("plain", "html", 1) + ).encode("utf-8") + content_type = "text/html; charset=utf-8" break elif atype[0] == "text/plain": break @@ -334,6 +368,137 @@ def materials_document(request, document, num=None, ext=None): return HttpResponseRedirect(redirect_to=doc.get_href(meeting=meeting)) +@requires_api_token("ietf.meeting.views.api_resolve_materials_name") +def api_resolve_materials_name_cached(request, document, num=None, ext=None): + """Resolve materials name into document to a blob spec + + Returns the bucket/name of a blob in the blob store that corresponds to the named + document. Handles resolution of revision if it is not specified and determines the + best extension if one is not provided. Response is JSON. + + As of 2025-10-10 we do not have blobs for all materials documents or for every + format of every document. This API still returns the bucket/name as if the blob + exists. Another API will allow the caller to obtain the file contents using that + name if it cannot be retrieved from the blob store. + """ + + def _error_response(status: int, detail: str): + return JsonResponse( + { + "status": status, + "title": "Error", + "detail": detail, + }, + status=status, + ) + + def _response(bucket: str, name: str): + return JsonResponse( + { + "bucket": bucket, + "name": name, + } + ) + + try: + resolved = ResolvedMaterial.objects.get( + meeting_number=num, name=document + ) + except ResolvedMaterial.DoesNotExist: + return _error_response( + HTTP_404_NOT_FOUND, f"No suitable file for {document} for meeting {num}" + ) + return _response(bucket=resolved.bucket, name=resolved.blob) + + +@requires_api_token +def api_retrieve_materials_blob(request, bucket, name): + """Retrieve contents of a meeting materials blob + + This is intended as a fallback if the web worker cannot retrieve a blob from + the blobstore itself. The most likely cause is retrieving an old materials document + that has not been backfilled. + + If a blob is requested that does not exist, this checks for it on the filesystem + and if found, adds it to the blobstore, creates a StoredObject record, and returns + the contents as it would have done if the blob was already present. + + As a special case, if a requested file with extension `.md.html` does not exist + but a file with the same name but extension `.md` does, `.md` file will be rendered + from markdown to html and returned / stored. + """ + DEFAULT_CONTENT_TYPES = { + ".html": "text/html;charset=utf-8", + ".md": "text/markdown;charset=utf-8", + ".pdf": "application/pdf", + ".txt": "text/plain;charset=utf-8", + } + + def _default_content_type(blob_name: str): + return DEFAULT_CONTENT_TYPES.get(Path(name).suffix, "application/octet-stream") + + if not ( + settings.ENABLE_BLOBSTORAGE + and bucket in settings.MATERIALS_TYPES_SERVED_BY_WORKER + ): + return HttpResponseNotFound(f"Bucket {bucket} not found.") + storage = storages[bucket] # if not configured, a server error will result + assert isinstance(storage, BlobdbStorage) + try: + blob = storage.open(name, "rb") + except FileNotFoundError: + pass + else: + # found the blob - return it + assert isinstance(blob, BlobFile) + log(f"Materials blob: directly returning {bucket}:{name}") + return FileResponse( + blob, + filename=name, + content_type=blob.content_type or _default_content_type(name), + ) + + # Did not find the blob. Create it if we can + name_as_path = Path(name) + if name_as_path.suffixes == [".md", ".html"]: + # special case: .md.html means we want to create the .md and the .md.html + # will come along as a bonus + name_to_store = name_as_path.stem # removes the .html + else: + name_to_store = name + + # See if we have a meeting-related document that matches the requested bucket and + # name. + try: + doc, rev = _get_materials_doc(Path(name_to_store).stem) + if doc.type_id != bucket: + raise Document.DoesNotExist + except Document.DoesNotExist: + log(f"Materials blob: no doc for {bucket}:{name}") + return HttpResponseNotFound( + f"Document corresponding to {bucket}:{name} not found." + ) + else: + # create all missing blobs for the doc while we're at it + log(f"Materials blob: storing blobs for {doc.name}-{doc.rev}") + store_blobs_for_one_material_doc(doc) + + # If we can make the blob at all, it now exists, so return it or a 404 + try: + blob = storage.open(name, "rb") + except FileNotFoundError: + log(f"Materials blob: no blob for {bucket}:{name}") + return HttpResponseNotFound(f"Object {bucket}:{name} not found.") + else: + # found the blob - return it + assert isinstance(blob, BlobFile) + return FileResponse( + blob, + filename=name, + content_type=blob.content_type or _default_content_type(name), + ) + + @login_required def materials_editable_groups(request, num=None): meeting = get_meeting(num) @@ -1510,6 +1675,11 @@ def list_schedules(request, num): class DiffSchedulesForm(forms.Form): from_schedule = forms.ChoiceField() to_schedule = forms.ChoiceField() + show_room_changes = forms.BooleanField( + initial=False, + required=False, + help_text="Include changes to room without a date or time change", + ) def __init__(self, meeting, user, *args, **kwargs): super().__init__(*args, **kwargs) @@ -1542,6 +1712,14 @@ def diff_schedules(request, num): raw_diffs = diff_meeting_schedules(from_schedule, to_schedule) diffs = prefetch_schedule_diff_objects(raw_diffs) + if not form.cleaned_data["show_room_changes"]: + # filter out room-only changes + diffs = [ + d + for d in diffs + if (d["change"] != "move") or (d["from"].time != d["to"].time) + ] + for d in diffs: s = d['session'] s.session_label = s.short_name @@ -1681,18 +1859,22 @@ def generate_agenda_data(num=None, force_refresh=False): :num: meeting number :force_refresh: True to force a refresh of the cache """ - cache = caches["default"] - cache_timeout = 6 * 60 - meeting = get_ietf_meeting(num) if meeting is None: raise Http404("No such full IETF meeting") elif int(meeting.number) <= 64: - return Http404("Pre-IETF 64 meetings are not available through this API") - else: - pass + raise Http404("Pre-IETF 64 meetings are not available through this API") + is_current_meeting = meeting.number == get_current_ietf_meeting_num() + + cache = caches["agenda"] + cache_timeout = ( + settings.AGENDA_CACHE_TIMEOUT_CURRENT_MEETING + if is_current_meeting + else settings.AGENDA_CACHE_TIMEOUT_DEFAULT + ) + cache_format = "1" # bump this on backward-incompatible data format changes - cache_key = f"generate_agenda_data_{meeting.number}" + cache_key = f"generate_agenda_data:{meeting.number}:v{cache_format}" if not force_refresh: cached_value = cache.get(cache_key) if cached_value is not None: @@ -1712,8 +1894,6 @@ def generate_agenda_data(num=None, force_refresh=False): filter_organizer = AgendaFilterOrganizer(assignments=filtered_assignments) - is_current_meeting = (num is None) or (num == get_current_ietf_meeting_num()) - # Get Floor Plans floors = FloorPlan.objects.filter(meeting=meeting).order_by('order') @@ -1788,21 +1968,32 @@ def api_get_session_materials(request, session_id=None): ) -def agenda_extract_schedule (item): +def agenda_extract_schedule(item): + if item.session.current_status == "resched": + resched_to = item.session.tombstone_for.official_timeslotassignment() + else: + resched_to = None return { "id": item.id, + "slug": item.slug(), "sessionId": item.session.id, - "room": item.room_name if item.timeslot.show_location else None, + "room": (item.timeslot.get_location() or None) if item.timeslot else None, "location": { "short": item.timeslot.location.floorplan.short, "name": item.timeslot.location.floorplan.name, } if (item.timeslot.show_location and item.timeslot.location and item.timeslot.location.floorplan) else {}, "acronym": item.acronym, - "duration": item.timeslot.duration.seconds, + "duration": item.timeslot.duration.total_seconds(), "name": item.session.name, + "slotId": item.timeslot.id, "slotName": item.timeslot.name, + "slotModified": item.timeslot.modified.isoformat(), "startDateTime": item.timeslot.time.isoformat(), "status": item.session.current_status, + "rescheduledTo": { + "startDateTime": resched_to.timeslot.time.isoformat(), + "duration": resched_to.timeslot.duration.total_seconds(), + } if resched_to is not None else {}, "type": item.session.type.slug, "purpose": item.session.purpose.slug, "isBoF": item.session.group_at_the_time().state_id == "bof", @@ -1820,7 +2011,7 @@ def agenda_extract_schedule (item): "showAgenda": True if (item.session.agenda() is not None or item.session.remote_instructions) else False }, "agenda": { - "url": item.session.agenda().get_href() + "url": item.session.agenda().get_versionless_href() } if item.session.agenda() is not None else { "url": None }, @@ -1921,7 +2112,7 @@ def slides_field(item): write_row(headings) - tz = datetime.timezone.utc if utc else schedule.meeting.tz() + tz = datetime.UTC if utc else schedule.meeting.tz() for item in filtered_assignments: row = [] row.append(item.timeslot.time.astimezone(tz).strftime("%Y-%m-%d")) @@ -2112,10 +2303,131 @@ def ical_session_status(assignment): else: return "CONFIRMED" + +def render_icalendar_precomp(agenda_data): + ical_content = generate_agenda_ical_precomp(agenda_data) + return HttpResponse(ical_content, content_type="text/calendar") + + def render_icalendar(schedule, assignments): ical_content = generate_agenda_ical(schedule, assignments) return HttpResponse(ical_content, content_type="text/calendar") + +def generate_agenda_ical_precomp(agenda_data): + """Generate iCalendar from precomputed data using the icalendar library""" + + cal = Calendar() + cal.add("prodid", "-//IETF//datatracker.ietf.org ical agenda//EN") + cal.add("version", "2.0") + cal.add("method", "PUBLISH") + + meeting_data = agenda_data["meeting"] + for item in agenda_data["schedule"]: + event = Event() + + uid = f"ietf-{meeting_data["number"]}-{item["slotId"]}-{item["acronym"]}" + event.add("uid", uid) + + # add custom field with meeting's local TZ + event.add("x-meeting-tz", meeting_data["timezone"]) + + if item["name"]: + summary = item["name"] + else: + summary = f"{item["groupAcronym"]} - {item["groupName"]}" + + if item["note"]: + summary += f" ({item["note"]})" + + event.add("summary", summary) + + if item["room"]: + event.add("location", item["room"]) # room name + + if item["status"] == "canceled": + status = "CANCELLED" + elif item["status"] == "resched": + resched_to = item["rescheduledTo"] + if resched_to is None: + status = "RESCHEDULED" + else: + resched_start = datetime.datetime.fromisoformat( + resched_to["startDateTime"] + ) + dur = datetime.timedelta(seconds=resched_to["duration"]) + resched_end = resched_start + dur + formatted_start = resched_start.strftime("%A %H:%M").upper() + formatted_end = resched_end.strftime("%H:%M") + status = f"RESCHEDULED TO {formatted_start}-{formatted_end}" + else: + status = "CONFIRMED" + event.add("status", status) + + event.add("class", "PUBLIC") + + start_time = datetime.datetime.fromisoformat(item["startDateTime"]) + duration = datetime.timedelta(seconds=item["duration"]) + event.add("dtstart", start_time) + event.add("dtend", start_time + duration) + + # DTSTAMP: when the event was created or last modified (in UTC) + # n.b. timeslot.modified may not be an accurate measure of this + event.add("dtstamp", datetime.datetime.fromisoformat(item["slotModified"])) + + description_parts = [item["slotName"]] + + if item["note"]: + description_parts.append(f"Note: {item["note"]}") + + links = item["links"] + if links["onsiteTool"]: + description_parts.append(f"Onsite tool: {links["onsiteTool"]}") + + if links["videoStream"]: + description_parts.append(f"Meetecho: {links["videoStream"]}") + + if links["webex"]: + description_parts.append(f"Webex: {links["webex"]}") + + if item["remoteInstructions"]: + description_parts.append( + f"Remote instructions: {item["remoteInstructions"]}" + ) + + try: + materials_url = absurl( + "ietf.meeting.views.session_details", + num=meeting_data["number"], + acronym=item["acronym"], + ) + except NoReverseMatch: + pass + else: + description_parts.append(f"Session materials: {materials_url}") + event.add("url", materials_url) + + if meeting_data["number"].isdigit(): + try: + agenda_url = absurl("agenda", num=meeting_data["number"]) + except NoReverseMatch: + pass + else: + description_parts.append(f"See in schedule: {agenda_url}#row-{item["slug"]}") + + if item["agenda"] and item["agenda"]["url"]: + description_parts.append(f"Agenda {item["agenda"]["url"]}") + + # Join all description parts with 2 newlines + description = "\n\n".join(description_parts) + event.add("description", description) + + # Add event to calendar + cal.add_component(event) + + return cal.to_ical().decode("utf-8") + + def generate_agenda_ical(schedule, assignments): """Generate iCalendar using the icalendar library""" @@ -2250,10 +2562,66 @@ def parse_agenda_filter_params(querydict): def should_include_assignment(filter_params, assignment): """Decide whether to include an assignment""" - shown = len(set(filter_params['show']).intersection(assignment.filter_keywords)) > 0 - hidden = len(set(filter_params['hide']).intersection(assignment.filter_keywords)) > 0 + if hasattr(assignment, "filter_keywords"): + kw = assignment.filter_keywords + elif isinstance(assignment, dict): + kw = assignment.get("filterKeywords", []) + else: + raise ValueError("Unsupported assignment instance") + shown = len(set(filter_params['show']).intersection(kw)) > 0 + hidden = len(set(filter_params['hide']).intersection(kw)) > 0 return shown and not hidden + +def agenda_ical_ietf(meeting, filt_params, acronym=None, session_id=None): + agenda_data = generate_agenda_data(meeting.number, force_refresh=False) + if acronym: + agenda_data["schedule"] = [ + item + for item in agenda_data["schedule"] + if item["groupAcronym"] == acronym + ] + elif session_id: + agenda_data["schedule"] = [ + item + for item in agenda_data["schedule"] + if item["sessionId"] == session_id + ] + if filt_params is not None: + # Apply the filter + agenda_data["schedule"] = [ + item + for item in agenda_data["schedule"] + if should_include_assignment(filt_params, item) + ] + return render_icalendar_precomp(agenda_data) + + +def agenda_ical_interim(meeting, filt_params, acronym=None, session_id=None): + schedule = get_schedule(meeting) + + if schedule is None and acronym is None and session_id is None: + raise Http404 + + assignments = SchedTimeSessAssignment.objects.filter( + schedule__in=[schedule, schedule.base], + session__on_agenda=True, + ) + assignments = preprocess_assignments_for_agenda(assignments, meeting) + AgendaKeywordTagger(assignments=assignments).apply() + + if filt_params is not None: + # Apply the filter + assignments = [a for a in assignments if should_include_assignment(filt_params, a)] + + if acronym: + assignments = [ a for a in assignments if a.session.group_at_the_time().acronym == acronym ] + elif session_id: + assignments = [ a for a in assignments if a.session_id == int(session_id) ] + + return render_icalendar(schedule, assignments) + + def agenda_ical(request, num=None, acronym=None, session_id=None): """Agenda ical view @@ -2281,33 +2649,20 @@ def agenda_ical(request, num=None, acronym=None, session_id=None): raise Http404 else: meeting = get_meeting(num, type_in=None) # get requested meeting, whatever its type - schedule = get_schedule(meeting) - if schedule is None and acronym is None and session_id is None: - raise Http404 - - assignments = SchedTimeSessAssignment.objects.filter( - schedule__in=[schedule, schedule.base], - session__on_agenda=True, - ) - assignments = preprocess_assignments_for_agenda(assignments, meeting) - AgendaKeywordTagger(assignments=assignments).apply() + if isinstance(session_id, str) and session_id.isdigit(): + session_id = int(session_id) try: filt_params = parse_agenda_filter_params(request.GET) except ValueError as e: return HttpResponseBadRequest(str(e)) - if filt_params is not None: - # Apply the filter - assignments = [a for a in assignments if should_include_assignment(filt_params, a)] - - if acronym: - assignments = [ a for a in assignments if a.session.group_at_the_time().acronym == acronym ] - elif session_id: - assignments = [ a for a in assignments if a.session_id == int(session_id) ] + if meeting.type_id == "ietf": + return agenda_ical_ietf(meeting, filt_params, acronym, session_id) + else: + return agenda_ical_interim(meeting, filt_params, acronym, session_id) - return render_icalendar(schedule, assignments) @cache_page(15 * 60) def agenda_json(request, num=None): @@ -2814,7 +3169,7 @@ def session_attendance(request, session_id, num): raise Http404("Bluesheets not found") cor_cut_off_date = session.meeting.get_submission_correction_date() - today_utc = date_today(datetime.timezone.utc) + today_utc = date_today(datetime.UTC) was_there = False can_add = False if request.user.is_authenticated: @@ -2949,6 +3304,7 @@ def upload_session_minutes(request, session_id, num): form.add_error(None, str(err)) else: # no exception -- success! + resolve_uploaded_material(meeting=session.meeting, doc=session.minutes()) messages.success(request, f'Successfully uploaded minutes as revision {session.minutes().rev}.') return redirect('ietf.meeting.views.session_details', num=num, acronym=session.group.acronym) else: @@ -3008,6 +3364,7 @@ def upload_session_narrativeminutes(request, session_id, num): form.add_error(None, str(err)) else: # no exception -- success! + resolve_uploaded_material(meeting=session.meeting, doc=session.narrative_minutes()) messages.success(request, f'Successfully uploaded narrative minutes as revision {session.narrative_minutes().rev}.') return redirect('ietf.meeting.views.session_details', num=num, acronym=session.group.acronym) else: @@ -3154,6 +3511,7 @@ def upload_session_agenda(request, session_id, num): form.add_error(None, save_error) else: doc.save_with_history([e]) + resolve_uploaded_material(meeting=session.meeting, doc=doc) messages.success(request, f'Successfully uploaded agenda as revision {doc.rev}.') return redirect('ietf.meeting.views.session_details',num=num,acronym=session.group.acronym) else: @@ -3337,6 +3695,7 @@ def upload_session_slides(request, session_id, num, name=None): else: doc.save_with_history([e]) post_process(doc) + resolve_uploaded_material(meeting=session.meeting, doc=doc) # Send MeetEcho updates even if we had a problem saving - that will keep it in sync with the # SessionPresentation, which was already saved regardless of problems saving the file. @@ -4248,21 +4607,137 @@ def upcoming_ical(request): else: ietfs = [] - meeting_vtz = {meeting.vtimezone() for meeting in meetings} - meeting_vtz.discard(None) - - # icalendar response file should have '\r\n' line endings per RFC5545 - response = render_to_string('meeting/upcoming.ics', { - 'vtimezones': ''.join(sorted(meeting_vtz)), - 'assignments': assignments, - 'ietfs': ietfs, - }, request=request) - response = parse_ical_line_endings(response) + response = render_upcoming_ical(assignments, ietfs, request) response = HttpResponse(response, content_type='text/calendar') response['Content-Disposition'] = 'attachment; filename="upcoming.ics"' return response +def render_important_dates_ical(meetings, request): + """Generate important dates using the icalendar library""" + cal = Calendar() + cal.add("prodid", "-//IETF//datatracker.ietf.org ical importantdates//EN") + cal.add("version", "2.0") + cal.add("method", "PUBLISH") + + for meeting in meetings: + for important_date in meeting.important_dates: + event = Event() + event.add("uid", f"ietf-{meeting.number}-{important_date.name_id}-" + f"{important_date.date.isoformat()}") + event.add("summary", f"IETF {meeting.number}: {important_date.name.name}") + event.add("class", "PUBLIC") + + if not important_date.midnight_cutoff: + event.add("dtstart", important_date.date) + else: + event.add("dtstart", datetime.datetime.combine( + important_date.date, + datetime.time(23, 59, 0, tzinfo=pytz.UTC)) + ) + + event.add("transp", "TRANSPARENT") + event.add("dtstamp", meeting.cached_updated) + description_lines = [important_date.name.desc] + if important_date.name.slug in ('openreg', 'earlybird'): + description_lines.append( + "Register here: https://www.ietf.org/how/meetings/register/") + if important_date.name.slug == 'opensched': + description_lines.append("To request a Working Group session, use the " + "IETF Meeting Session Request Tool:") + description_lines.append(f"{request.scheme}://{request.get_host()}" + f"{reverse('ietf.meeting.views_session_request.list_view')}") + description_lines.append("If you are working on a BOF request, it is " + "highly recommended to tell the IESG") + description_lines.append("now by sending an email to iesg@ietf.org " + "to get advance help with the request.") + if important_date.name.slug == 'cutoffwgreq': + description_lines.append("To request a Working Group session, use the " + "IETF Meeting Session Request Tool:") + description_lines.append(f"{request.scheme}://{request.get_host()}" + f"{reverse('ietf.meeting.views_session_request.list_view')}") + if important_date.name.slug == 'cutoffbofreq': + description_lines.append("To request a BOF, please see instructions on " + "Requesting a BOF:") + description_lines.append("https://www.ietf.org/how/bofs/bof-procedures/") + if important_date.name.slug == 'idcutoff': + description_lines.append("Upload using the I-D Submission Tool:") + description_lines.append(f"{request.scheme}://{request.get_host()}" + f"{reverse('ietf.submit.views.upload_submission')}") + if important_date.name.slug in ( + 'draftwgagenda', + 'revwgagenda', + 'procsub', + 'revslug' + ): + description_lines.append("Upload using the Meeting Materials " + "Management Tool:") + description_lines.append(f"{request.scheme}://{request.get_host()}" + f"{reverse('ietf.meeting.views.materials', + kwargs={'num': meeting.number})}") + + event.add("description", "\n".join(description_lines)) + cal.add_component(event) + + return cal.to_ical().decode("utf-8") + +def render_upcoming_ical(assignments, meetings, request): + """Generate upcoming using the icalendar library""" + + cal = Calendar() + cal.add("prodid", "-//IETF//datatracker.ietf.org ical upcoming//EN") + cal.add("version", "2.0") + cal.add("method", "PUBLISH") + + for item in assignments: + event = Event() + + event.add("uid", f"ietf-{item.session.meeting.number}-{item.timeslot.pk}") + event.add("summary", f"{item.session.group.acronym.lower()} - {item.session.name if item.session.name else item.session.group.name}") + + if item.schedule.meeting.city: + event.add("location", f"{item.schedule.meeting.city},{item.schedule.meeting.country}") + + event.add("status", item.session.ical_status) + event.add("class", "PUBLIC") + + event.add("dtstart", item.timeslot.utc_start_time()) + event.add("dtend", item.timeslot.utc_end_time()) + event.add("dtstamp", item.timeslot.modified) + if item.session.agenda(): + event.add("url", item.session.agenda().get_href()) + + description_lines = [] + if item.timeslot.name: + description_lines.append(f"{item.timeslot.name}") + if item.session.agenda_note: + description_lines.append(f"Note: {item.session.agenda_note}") + + for material in item.session.materials.all(): + title_part = f" ({material.title})" if material.type.name != "Agenda" else "" + description_lines.append(f"{material.type}{title_part}: {material.get_href()}") + + if item.session.remote_instructions: + description_lines.append(f"Remote instructions: {item.session.remote_instructions}") + + event.add("description", "\n".join(description_lines)) + cal.add_component(event) + + for meeting in meetings: + event = Event() + event.add("uid", f"ietf-{meeting.number}") + event.add("summary", f"IETF {meeting.number}") + if meeting.city: + event.add("location", f"{meeting.city},{meeting.country}") + event.add("class", "PUBLIC") + event.add("dtstart", meeting.date) + event.add("dtend", meeting.end_date() + datetime.timedelta(days=1)) + event.add("dtstamp", meeting.cached_updated) + event.add("url", f"{request.scheme}://{request.get_host()}{reverse('agenda', kwargs={'num': meeting.number})}") + + cal.add_component(event) + + return cal.to_ical().decode("utf-8") def upcoming_json(request): '''Return Upcoming meetings in json format''' @@ -4737,6 +5212,7 @@ def err(code, text): write_doc_for_session(session, 'chatlog', filename, json.dumps(apidata['chatlog'])) e = NewRevisionDocEvent.objects.create(doc=doc, rev=doc.rev, by=request.user.person, type='new_revision', desc='New revision available: %s'%doc.rev) doc.save_with_history([e]) + resolve_uploaded_material(meeting=session.meeting, doc=doc) return HttpResponse( "Done", status=200, @@ -4785,6 +5261,7 @@ def err(code, text): write_doc_for_session(session, 'polls', filename, json.dumps(apidata['polls'])) e = NewRevisionDocEvent.objects.create(doc=doc, rev=doc.rev, by=request.user.person, type='new_revision', desc='New revision available: %s'%doc.rev) doc.save_with_history([e]) + resolve_uploaded_material(meeting=session.meeting, doc=doc) return HttpResponse( "Done", status=200, @@ -4879,11 +5356,8 @@ def important_dates(request, num=None, output_format=None): if output_format == 'ics': preprocess_meeting_important_dates(meetings) - ics = render_to_string('meeting/important_dates.ics', { - 'meetings': meetings, - }, request=request) - # icalendar response file should have '\r\n' line endings per RFC5545 - response = HttpResponse(parse_ical_line_endings(ics), content_type='text/calendar') + response = HttpResponse(render_important_dates_ical(meetings, request), + content_type='text/calendar') response['Content-Disposition'] = 'attachment; filename="important-dates.ics"' return response @@ -5120,7 +5594,7 @@ def approve_proposed_slides(request, slidesubmission_id, num): if request.POST.get('approve'): # Ensure that we have a file to approve. The system gets cranky otherwise. if submission.filename is None or submission.filename == '' or not os.path.isfile(submission.staged_filepath()): - return HttpResponseNotFound("The slides you attempted to approve could not be found. Please disapprove and delete them instead.") + return HttpResponseNotFound("The slides you attempted to approve could not be found. Please decline and delete them instead.") title = form.cleaned_data['title'] if existing_doc: doc = Document.objects.get(name=name) @@ -5167,6 +5641,7 @@ def approve_proposed_slides(request, slidesubmission_id, num): doc.store_bytes(target_filename, retrieve_bytes("staging", submission.filename)) remove_from_storage("staging", submission.filename) post_process(doc) + resolve_uploaded_material(meeting=submission.session.meeting, doc=doc) DocEvent.objects.create(type="approved_slides", doc=doc, rev=doc.rev, by=request.user.person, desc="Slides approved") # update meetecho slide info if configured @@ -5235,6 +5710,52 @@ def approve_proposed_slides(request, slidesubmission_id, num): }) +@role_required("Secretariat") +def notify_meetecho_of_all_slides(request, num, acronym): + """Notify meetecho of state of all slides for the group + + Respects the usual notification window around each session. Meetecho will ignore + notices outside that window anyway, so no sense sending them. + """ + meeting = get_meeting(num=num, type_in=None) # raises 404 + if request.method != "POST": + return HttpResponseNotAllowed( + content="Method not allowed", + content_type=f"text/plain; charset={settings.DEFAULT_CHARSET}", + permitted_methods=("POST",), + ) + scheduled_sessions = [ + session + for session in get_sessions(meeting.number, acronym) + if session.current_status == "sched" + ] + sm = SlidesManager(api_config=settings.MEETECHO_API_CONFIG) + updated = [] + for session in scheduled_sessions: + if sm.send_update(session): + updated.append(session) + if len(updated) > 0: + messages.success( + request, + f"Notified Meetecho about slides for {','.join(str(s) for s in updated)}", + ) + elif sm.slides_notify_time is not None: + messages.warning( + request, + "No sessions were eligible for Meetecho slides update. Updates are " + f"only sent within {sm.slides_notify_time} before or after the session.", + ) + else: + messages.warning( + request, + "No sessions were eligible for Meetecho slides update. Updates are " + "currently disabled.", + ) + return redirect( + "ietf.meeting.views.session_details", num=meeting.number, acronym=acronym + ) + + def import_session_minutes(request, session_id, num): """Import session minutes from the ietf.notes.org site @@ -5282,6 +5803,7 @@ def import_session_minutes(request, session_id, num): except SaveMaterialsError as err: form.add_error(None, str(err)) else: + resolve_uploaded_material(meeting=session.meeting, doc=session.minutes()) messages.success(request, f'Successfully imported minutes as revision {session.minutes().rev}.') return redirect('ietf.meeting.views.session_details', num=num, acronym=session.group.acronym) else: diff --git a/ietf/secr/sreq/views.py b/ietf/meeting/views_session_request.py similarity index 80% rename from ietf/secr/sreq/views.py rename to ietf/meeting/views_session_request.py index eb93168e1c..a1ef74f1b8 100644 --- a/ietf/secr/sreq/views.py +++ b/ietf/meeting/views_session_request.py @@ -1,29 +1,26 @@ -# Copyright The IETF Trust 2013-2022, All Rights Reserved +# Copyright The IETF Trust 2007-2025, All Rights Reserved # -*- coding: utf-8 -*- - import datetime import inflect from collections import defaultdict, OrderedDict from django.conf import settings from django.contrib import messages +from django.core.exceptions import ObjectDoesNotExist from django.db.models import Q from django.shortcuts import render, get_object_or_404, redirect from django.http import Http404 -import debug # pyflakes:ignore - from ietf.group.models import Group, GroupFeatures from ietf.ietfauth.utils import has_role, role_required -from ietf.meeting.models import Meeting, Session, Constraint, ResourceAssociation, SchedulingEvent from ietf.meeting.helpers import get_meeting +from ietf.meeting.models import Session, Meeting, Constraint, ResourceAssociation, SchedulingEvent from ietf.meeting.utils import add_event_info_to_session_qs -from ietf.name.models import SessionStatusName, ConstraintName -from ietf.secr.sreq.forms import (SessionForm, ToolStatusForm, allowed_conflicting_groups, +from ietf.meeting.forms import (SessionRequestStatusForm, SessionRequestForm, allowed_conflicting_groups, JOINT_FOR_SESSION_CHOICES) +from ietf.name.models import SessionStatusName, ConstraintName from ietf.secr.utils.decorators import check_permissions -from ietf.secr.utils.group import get_my_groups from ietf.utils.mail import send_mail from ietf.mailtrigger.utils import gather_address_lists @@ -31,12 +28,25 @@ # Globals # ------------------------------------------------- # TODO: This needs to be replaced with something that pays attention to groupfeatures -AUTHORIZED_ROLES=('WG Chair','WG Secretary','RG Chair','IAB Group Chair','Area Director','Secretariat','Team Chair','IRTF Chair','Program Chair','Program Lead','Program Secretary', 'EDWG Chair') +AUTHORIZED_ROLES = ( + 'WG Chair', + 'WG Secretary', + 'RG Chair', + 'IAB Group Chair', + 'Area Director', + 'Secretariat', + 'Team Chair', + 'IRTF Chair', + 'Program Chair', + 'Program Lead', + 'Program Secretary', + 'EDWG Chair') # ------------------------------------------------- # Helper Functions # ------------------------------------------------- + def check_app_locked(meeting=None): ''' This function returns True if the application is locked to non-secretariat users. @@ -45,6 +55,54 @@ def check_app_locked(meeting=None): meeting = get_meeting(days=14) return bool(meeting.session_request_lock_message) + +def get_lock_message(meeting=None): + ''' + Returns the message to display to non-secretariat users when the tool is locked. + ''' + if not meeting: + meeting = get_meeting(days=14) + return meeting.session_request_lock_message + + +def get_my_groups(user, conclude=False): + ''' + Takes a Django user object (from request) + Returns a list of groups the user has access to. Rules are as follows + secretariat - has access to all groups + area director - has access to all groups in their area + wg chair or secretary - has access to their own group + chair of irtf has access to all irtf groups + + If user=None than all groups are returned. + concluded=True means include concluded groups. Need this to upload materials for groups + after they've been concluded. it happens. + ''' + my_groups = set() + states = ['bof', 'proposed', 'active'] + if conclude: + states.extend(['conclude', 'bof-conc']) + + all_groups = Group.objects.filter(type__features__has_meetings=True, state__in=states).order_by('acronym') + if user is None or has_role(user, 'Secretariat'): + return all_groups + + try: + person = user.person + except ObjectDoesNotExist: + return list() + + for group in all_groups: + if group.role_set.filter(person=person, name__in=('chair', 'secr', 'ad')): + my_groups.add(group) + continue + if group.parent and group.parent.role_set.filter(person=person, name__in=('ad', 'chair')): + my_groups.add(group) + continue + + return list(my_groups) + + def get_initial_session(sessions, prune_conflicts=False): ''' This function takes a queryset of sessions ordered by 'id' for consistency. It returns @@ -97,13 +155,43 @@ def valid_conflict(conflict): initial['joint_for_session_display'] = dict(JOINT_FOR_SESSION_CHOICES)[initial['joint_for_session']] return initial -def get_lock_message(meeting=None): + +def inbound_session_conflicts_as_string(group, meeting): ''' - Returns the message to display to non-secretariat users when the tool is locked. + Takes a Group object and Meeting object and returns a string of other groups which have + a conflict with this one ''' - if not meeting: - meeting = get_meeting(days=14) - return meeting.session_request_lock_message + constraints = group.constraint_target_set.filter(meeting=meeting, name__is_group_conflict=True) + group_set = set(constraints.values_list('source__acronym', flat=True)) # set to de-dupe + group_list = sorted(group_set) # give a consistent order + return ', '.join(group_list) + + +def get_outbound_conflicts(form: SessionRequestForm): + """extract wg conflict constraint data from a SessionForm""" + outbound_conflicts = [] + for conflictname, cfield_id in form.wg_constraint_field_ids(): + conflict_groups = form.cleaned_data[cfield_id] + if len(conflict_groups) > 0: + outbound_conflicts.append(dict(name=conflictname, groups=conflict_groups)) + return outbound_conflicts + + +def save_conflicts(group, meeting, conflicts, name): + ''' + This function takes a Group, Meeting a string which is a list of Groups acronyms (conflicts), + and the constraint name (conflict|conflic2|conflic3) and creates Constraint records + ''' + constraint_name = ConstraintName.objects.get(slug=name) + acronyms = conflicts.replace(',',' ').split() + for acronym in acronyms: + target = Group.objects.get(acronym=acronym) + + constraint = Constraint(source=group, + target=target, + meeting=meeting, + name=constraint_name) + constraint.save() def get_requester_text(person, group): @@ -129,22 +217,6 @@ def get_requester_text(person, group): ) -def save_conflicts(group, meeting, conflicts, name): - ''' - This function takes a Group, Meeting a string which is a list of Groups acronyms (conflicts), - and the constraint name (conflict|conflic2|conflic3) and creates Constraint records - ''' - constraint_name = ConstraintName.objects.get(slug=name) - acronyms = conflicts.replace(',',' ').split() - for acronym in acronyms: - target = Group.objects.get(acronym=acronym) - - constraint = Constraint(source=group, - target=target, - meeting=meeting, - name=constraint_name) - constraint.save() - def send_notification(group, meeting, login, sreq_data, session_data, action): ''' This function generates email notifications for various session request activities. @@ -152,10 +224,10 @@ def send_notification(group, meeting, login, sreq_data, session_data, action): session_data is an array of data from individual session subforms action argument is a string [new|update]. ''' - (to_email, cc_list) = gather_address_lists('session_requested',group=group,person=login) + (to_email, cc_list) = gather_address_lists('session_requested', group=group, person=login) from_email = (settings.SESSION_REQUEST_FROM_EMAIL) subject = '%s - New Meeting Session Request for IETF %s' % (group.acronym, meeting.number) - template = 'sreq/session_request_notification.txt' + template = 'meeting/session_request_notification.txt' # send email context = {} @@ -164,7 +236,7 @@ def send_notification(group, meeting, login, sreq_data, session_data, action): context['meeting'] = meeting context['login'] = login context['header'] = 'A new' - context['requester'] = get_requester_text(login,group) + context['requester'] = get_requester_text(login, group) # update overrides if action == 'update': @@ -174,10 +246,10 @@ def send_notification(group, meeting, login, sreq_data, session_data, action): # if third session requested approval is required # change headers TO=ADs, CC=session-request, submitter and cochairs if len(session_data) > 2: - (to_email, cc_list) = gather_address_lists('session_requested_long',group=group,person=login) + (to_email, cc_list) = gather_address_lists('session_requested_long', group=group, person=login) subject = '%s - Request for meeting session approval for IETF %s' % (group.acronym, meeting.number) - template = 'sreq/session_approval_notification.txt' - #status_text = 'the %s Directors for approval' % group.parent + template = 'meeting/session_approval_notification.txt' + # status_text = 'the %s Directors for approval' % group.parent context['session_lengths'] = [sd['requested_duration'] for sd in session_data] @@ -189,103 +261,188 @@ def send_notification(group, meeting, login, sreq_data, session_data, action): context, cc=cc_list) -def inbound_session_conflicts_as_string(group, meeting): - ''' - Takes a Group object and Meeting object and returns a string of other groups which have - a conflict with this one - ''' - constraints = group.constraint_target_set.filter(meeting=meeting, name__is_group_conflict=True) - group_set = set(constraints.values_list('source__acronym', flat=True)) # set to de-dupe - group_list = sorted(group_set) # give a consistent order - return ', '.join(group_list) + +def session_changed(session): + latest_event = SchedulingEvent.objects.filter(session=session).order_by('-time', '-id').first() + + if latest_event and latest_event.status_id == "schedw" and session.meeting.schedule is not None: + # send an email to iesg-secretariat to alert to change + pass + + +def status_slug_for_new_session(session, session_number): + if session.group.features.acts_like_wg and session_number == 2: + return 'apprw' + return 'schedw' # ------------------------------------------------- # View Functions # ------------------------------------------------- -@check_permissions -def approve(request, acronym): + + +@role_required(*AUTHORIZED_ROLES) +def list_view(request): ''' - This view approves the third session. For use by ADs or Secretariat. + Display list of groups the user has access to. ''' meeting = get_meeting(days=14) - group = get_object_or_404(Group, acronym=acronym) - session = add_event_info_to_session_qs(Session.objects.filter(meeting=meeting, group=group)).filter(current_status='apprw').first() - if session is None: - raise Http404 + # check for locked flag + is_locked = check_app_locked() + if is_locked and not has_role(request.user, 'Secretariat'): + message = get_lock_message() + return render(request, 'meeting/session_request_locked.html', { + 'message': message, + 'meeting': meeting}) - if has_role(request.user,'Secretariat') or group.parent.role_set.filter(name='ad',person=request.user.person): - SchedulingEvent.objects.create( - session=session, - status=SessionStatusName.objects.get(slug='appr'), - by=request.user.person, - ) - session_changed(session) + scheduled_groups = [] + unscheduled_groups = [] - messages.success(request, 'Third session approved') - return redirect('ietf.secr.sreq.views.view', acronym=acronym) - else: - # if an unauthorized user gets here return error - messages.error(request, 'Not authorized to approve the third session') - return redirect('ietf.secr.sreq.views.view', acronym=acronym) + group_types = GroupFeatures.objects.filter(has_meetings=True).values_list('type', flat=True) -@check_permissions -def cancel(request, acronym): - ''' - This view cancels a session request and sends a notification. - To cancel, or withdraw the request set status = deleted. - "canceled" status is used by the secretariat. + my_groups = [g for g in get_my_groups(request.user, conclude=True) if g.type_id in group_types] - NOTE: this function can also be called after a session has been - scheduled during the period when the session request tool is - reopened. In this case be sure to clear the timeslot assignment as well. + sessions_by_group = defaultdict(list) + for s in add_event_info_to_session_qs(Session.objects.filter(meeting=meeting, group__in=my_groups)).filter(current_status__in=['schedw', 'apprw', 'appr', 'sched']): + sessions_by_group[s.group_id].append(s) + + for group in my_groups: + group.meeting_sessions = sessions_by_group.get(group.pk, []) + + if group.pk in sessions_by_group: + # include even if concluded as we need to to see that the + # sessions are there + scheduled_groups.append(group) + else: + if group.state_id not in ['conclude', 'bof-conc']: + # too late for unscheduled if concluded + unscheduled_groups.append(group) + + # warn if there are no associated groups + if not scheduled_groups and not unscheduled_groups: + messages.warning(request, 'The account %s is not associated with any groups. If you have multiple Datatracker accounts you may try another or report a problem to %s' % (request.user, settings.SECRETARIAT_ACTION_EMAIL)) + + # add session status messages for use in template + for group in scheduled_groups: + if not group.features.acts_like_wg or (len(group.meeting_sessions) < 3): + group.status_message = group.meeting_sessions[0].current_status + else: + group.status_message = 'First two sessions: %s, Third session: %s' % (group.meeting_sessions[0].current_status, group.meeting_sessions[2].current_status) + + # add not meeting indicators for use in template + for group in unscheduled_groups: + if any(s.current_status == 'notmeet' for s in group.meeting_sessions): + group.not_meeting = True + + return render(request, 'meeting/session_request_list.html', { + 'is_locked': is_locked, + 'meeting': meeting, + 'scheduled_groups': scheduled_groups, + 'unscheduled_groups': unscheduled_groups}, + ) + + +@role_required('Secretariat') +def status(request): + ''' + This view handles locking and unlocking of the session request tool to the public. ''' meeting = get_meeting(days=14) - group = get_object_or_404(Group, acronym=acronym) - sessions = Session.objects.filter(meeting=meeting,group=group).order_by('id') - login = request.user.person + is_locked = check_app_locked(meeting=meeting) - # delete conflicts - Constraint.objects.filter(meeting=meeting,source=group).delete() + if request.method == 'POST': + button_text = request.POST.get('submit', '') + if button_text == 'Back': + return redirect('ietf.meeting.views_session_request.list_view') - # mark sessions as deleted - for session in sessions: - SchedulingEvent.objects.create( - session=session, - status=SessionStatusName.objects.get(slug='deleted'), - by=request.user.person, - ) - session_changed(session) + form = SessionRequestStatusForm(request.POST) - # clear schedule assignments if already scheduled - session.timeslotassignments.all().delete() + if button_text == 'Lock': + if form.is_valid(): + meeting.session_request_lock_message = form.cleaned_data['message'] + meeting.save() + messages.success(request, 'Session Request Tool is now Locked') + return redirect('ietf.meeting.views_session_request.list_view') - # send notifitcation - (to_email, cc_list) = gather_address_lists('session_request_cancelled',group=group,person=login) - from_email = (settings.SESSION_REQUEST_FROM_EMAIL) - subject = '%s - Cancelling a meeting request for IETF %s' % (group.acronym, meeting.number) - send_mail(request, to_email, from_email, subject, 'sreq/session_cancel_notification.txt', - {'requester':get_requester_text(login,group), - 'meeting':meeting}, cc=cc_list) + elif button_text == 'Unlock': + meeting.session_request_lock_message = '' + meeting.save() + messages.success(request, 'Session Request Tool is now Unlocked') + return redirect('ietf.meeting.views_session_request.list_view') - messages.success(request, 'The %s Session Request has been cancelled' % group.acronym) - return redirect('ietf.secr.sreq.views.main') + else: + if is_locked: + message = get_lock_message() + initial = {'message': message} + form = SessionRequestStatusForm(initial=initial) + else: + form = SessionRequestStatusForm() + return render(request, 'meeting/session_request_status.html', { + 'is_locked': is_locked, + 'form': form}, + ) -def status_slug_for_new_session(session, session_number): - if session.group.features.acts_like_wg and session_number == 2: - return 'apprw' - return 'schedw' +@check_permissions +def new_request(request, acronym): + ''' + This view gathers details for a new session request. The user proceeds to confirm() + to create the request. + ''' + group = get_object_or_404(Group, acronym=acronym) + if len(group.features.session_purposes) == 0: + raise Http404(f'Cannot request sessions for group "{acronym}"') + meeting = get_meeting(days=14) + session_conflicts = dict(inbound=inbound_session_conflicts_as_string(group, meeting)) -def get_outbound_conflicts(form: SessionForm): - """extract wg conflict constraint data from a SessionForm""" - outbound_conflicts = [] - for conflictname, cfield_id in form.wg_constraint_field_ids(): - conflict_groups = form.cleaned_data[cfield_id] - if len(conflict_groups) > 0: - outbound_conflicts.append(dict(name=conflictname, groups=conflict_groups)) - return outbound_conflicts + # check if app is locked + is_locked = check_app_locked() + if is_locked and not has_role(request.user, 'Secretariat'): + messages.warning(request, "The Session Request Tool is closed") + return redirect('ietf.meeting.views_session_request.list_view') + + if request.method == 'POST': + button_text = request.POST.get('submit', '') + if button_text == 'Cancel': + return redirect('ietf.meeting.views_session_request.list_view') + + form = SessionRequestForm(group, meeting, request.POST, notifications_optional=has_role(request.user, "Secretariat")) + if form.is_valid(): + return confirm(request, acronym) + + # the "previous" querystring causes the form to be returned + # pre-populated with data from last meeeting's session request + elif request.method == 'GET' and 'previous' in request.GET: + latest_session = add_event_info_to_session_qs(Session.objects.filter(meeting__type_id='ietf', group=group)).exclude(current_status__in=['notmeet', 'deleted', 'canceled',]).order_by('-meeting__date').first() + if latest_session: + previous_meeting = Meeting.objects.get(number=latest_session.meeting.number) + previous_sessions = add_event_info_to_session_qs(Session.objects.filter(meeting=previous_meeting, group=group)).exclude(current_status__in=['notmeet', 'deleted']).order_by('id') + if not previous_sessions: + messages.warning(request, 'This group did not meet at %s' % previous_meeting) + return redirect('ietf.meeting.views_session_request.new_request', acronym=acronym) + else: + messages.info(request, 'Fetched session info from %s' % previous_meeting) + else: + messages.warning(request, 'Did not find any previous meeting') + return redirect('ietf.meeting.views_session_request.new_request', acronym=acronym) + + initial = get_initial_session(previous_sessions, prune_conflicts=True) + if 'resources' in initial: + initial['resources'] = [x.pk for x in initial['resources']] + form = SessionRequestForm(group, meeting, initial=initial, notifications_optional=has_role(request.user, "Secretariat")) + + else: + initial = {} + form = SessionRequestForm(group, meeting, initial=initial, notifications_optional=has_role(request.user, "Secretariat")) + + return render(request, 'meeting/session_request_form.html', { + 'meeting': meeting, + 'form': form, + 'group': group, + 'is_create': True, + 'session_conflicts': session_conflicts}, + ) @role_required(*AUTHORIZED_ROLES) @@ -295,11 +452,11 @@ def confirm(request, acronym): to confirm for submission. ''' # FIXME: this should be using form.is_valid/form.cleaned_data - invalid input will make it crash - group = get_object_or_404(Group,acronym=acronym) + group = get_object_or_404(Group, acronym=acronym) if len(group.features.session_purposes) == 0: raise Http404(f'Cannot request sessions for group "{acronym}"') meeting = get_meeting(days=14) - form = SessionForm(group, meeting, request.POST, hidden=True, notifications_optional=has_role(request.user, "Secretariat")) + form = SessionRequestForm(group, meeting, request.POST, hidden=True, notifications_optional=has_role(request.user, "Secretariat")) form.is_valid() login = request.user.person @@ -307,8 +464,8 @@ def confirm(request, acronym): # check if request already exists for this group if add_event_info_to_session_qs(Session.objects.filter(group=group, meeting=meeting)).filter(Q(current_status__isnull=True) | ~Q(current_status__in=['deleted', 'notmeet'])): messages.warning(request, 'Sessions for working group %s have already been requested once.' % group.acronym) - return redirect('ietf.secr.sreq.views.main') - + return redirect('ietf.meeting.views_session_request.list_view') + session_data = form.data.copy() # use cleaned_data for the 'bethere' field so we get the Person instances session_data['bethere'] = form.cleaned_data['bethere'] if 'bethere' in form.cleaned_data else [] @@ -318,7 +475,7 @@ def confirm(request, acronym): session_data['joint_for_session_display'] = dict(JOINT_FOR_SESSION_CHOICES)[session_data['joint_for_session']] if form.cleaned_data.get('timeranges'): session_data['timeranges_display'] = [t.desc for t in form.cleaned_data['timeranges']] - session_data['resources'] = [ ResourceAssociation.objects.get(pk=pk) for pk in request.POST.getlist('resources') ] + session_data['resources'] = [ResourceAssociation.objects.get(pk=pk) for pk in request.POST.getlist('resources')] # extract wg conflict constraint data for the view / notifications outbound_conflicts = get_outbound_conflicts(form) @@ -326,7 +483,7 @@ def confirm(request, acronym): button_text = request.POST.get('submit', '') if button_text == 'Cancel': messages.success(request, 'Session Request has been cancelled') - return redirect('ietf.secr.sreq.views.main') + return redirect('ietf.meeting.views_session_request.list_view') if request.method == 'POST' and button_text == 'Submit': # delete any existing session records with status = canceled or notmeet @@ -344,10 +501,10 @@ def confirm(request, acronym): if 'resources' in form.data: new_session.resources.set(session_data['resources']) jfs = form.data.get('joint_for_session', '-1') - if not jfs: # jfs might be '' + if not jfs: # jfs might be '' jfs = '-1' if int(jfs) == count + 1: # count is zero-indexed - groups_split = form.cleaned_data.get('joint_with_groups').replace(',',' ').split() + groups_split = form.cleaned_data.get('joint_with_groups').replace(',', ' ').split() joint = Group.objects.filter(acronym__in=groups_split) new_session.joint_with_groups.set(joint) new_session.save() @@ -388,36 +545,105 @@ def confirm(request, acronym): 'new', ) - status_text = 'IETF Agenda to be scheduled' - messages.success(request, 'Your request has been sent to %s' % status_text) - return redirect('ietf.secr.sreq.views.main') + status_text = 'IETF Agenda to be scheduled' + messages.success(request, 'Your request has been sent to %s' % status_text) + return redirect('ietf.meeting.views_session_request.list_view') + + # POST from request submission + session_conflicts = dict( + outbound=outbound_conflicts, # each is a dict with name and groups as keys + inbound=inbound_session_conflicts_as_string(group, meeting), + ) + if form.cleaned_data.get('third_session'): + messages.warning(request, 'Note: Your request for a third session must be approved by an area director before being submitted to agenda@ietf.org. Click "Submit" below to email an approval request to the area directors') + + return render(request, 'meeting/session_request_confirm.html', { + 'form': form, + 'session': session_data, + 'group': group, + 'meeting': meeting, + 'session_conflicts': session_conflicts}, + ) + + +@role_required(*AUTHORIZED_ROLES) +def view_request(request, acronym, num=None): + ''' + This view displays the session request info + ''' + meeting = get_meeting(num, days=14) + group = get_object_or_404(Group, acronym=acronym) + query = Session.objects.filter(meeting=meeting, group=group) + status_is_null = Q(current_status__isnull=True) + status_allowed = ~Q(current_status__in=("canceled", "notmeet", "deleted")) + sessions = ( + add_event_info_to_session_qs(query) + .filter(status_is_null | status_allowed) + .order_by("id") + ) + + # check if app is locked + is_locked = check_app_locked() + if is_locked: + messages.warning(request, "The Session Request Tool is closed") + + # if there are no session requests yet, redirect to new session request page + if not sessions: + if is_locked: + return redirect('ietf.meeting.views_session_request.list_view') + else: + return redirect('ietf.meeting.views_session_request.new_request', acronym=acronym) + + activities = [{ + 'act_date': e.time.strftime('%b %d, %Y'), + 'act_time': e.time.strftime('%H:%M:%S'), + 'activity': e.status.name, + 'act_by': e.by, + } for e in sessions[0].schedulingevent_set.select_related('status', 'by')] + + # gather outbound conflicts + outbound_dict = OrderedDict() + for obc in group.constraint_source_set.filter(meeting=meeting, name__is_group_conflict=True): + if obc.name.slug not in outbound_dict: + outbound_dict[obc.name.slug] = [] + outbound_dict[obc.name.slug].append(obc.target.acronym) - # POST from request submission session_conflicts = dict( - outbound=outbound_conflicts, # each is a dict with name and groups as keys inbound=inbound_session_conflicts_as_string(group, meeting), + outbound=[dict(name=ConstraintName.objects.get(slug=slug), groups=' '.join(groups)) + for slug, groups in outbound_dict.items()], ) - return render(request, 'sreq/confirm.html', { - 'form': form, - 'session': session_data, + + show_approve_button = False + + # if sessions include a 3rd session waiting approval and the user is a secretariat or AD of the group + # display approve button + if any(s.current_status == 'apprw' for s in sessions): + if has_role(request.user, 'Secretariat') or group.parent.role_set.filter(name='ad', person=request.user.person): + show_approve_button = True + + # build session dictionary (like querydict from new session request form) for use in template + session = get_initial_session(sessions) + + return render(request, 'meeting/session_request_view.html', { + 'can_edit': (not is_locked) or has_role(request.user, 'Secretariat'), + 'can_cancel': (not is_locked) or has_role(request.user, 'Secretariat'), + 'session': session, # legacy processed data + 'sessions': sessions, # actual session instances + 'activities': activities, + 'meeting': meeting, 'group': group, - 'session_conflicts': session_conflicts}, + 'session_conflicts': session_conflicts, + 'show_approve_button': show_approve_button}, ) - -def session_changed(session): - latest_event = SchedulingEvent.objects.filter(session=session).order_by('-time', '-id').first() - - if latest_event and latest_event.status_id == "schedw" and session.meeting.schedule != None: - # send an email to iesg-secretariat to alert to change - pass @check_permissions -def edit(request, acronym, num=None): +def edit_request(request, acronym, num=None): ''' This view allows the user to edit details of the session request ''' - meeting = get_meeting(num,days=14) + meeting = get_meeting(num, days=14) group = get_object_or_404(Group, acronym=acronym) if len(group.features.session_purposes) == 0: raise Http404(f'Cannot request sessions for group "{acronym}"') @@ -443,15 +669,15 @@ def edit(request, acronym, num=None): login = request.user.person first_session = Session() - if(len(sessions) > 0): + if (len(sessions) > 0): first_session = sessions[0] if request.method == 'POST': button_text = request.POST.get('submit', '') if button_text == 'Cancel': - return redirect('ietf.secr.sreq.views.view', acronym=acronym) + return redirect('ietf.meeting.views_session_request.view_request', acronym=acronym) - form = SessionForm(group, meeting, request.POST, initial=initial, notifications_optional=has_role(request.user, "Secretariat")) + form = SessionRequestForm(group, meeting, request.POST, initial=initial, notifications_optional=has_role(request.user, "Secretariat")) if form.is_valid(): if form.has_changed(): changed_session_forms = [sf for sf in form.session_forms.forms_to_keep if sf.has_changed()] @@ -513,11 +739,11 @@ def edit(request, acronym, num=None): if 'resources' in form.changed_data: new_resource_ids = form.cleaned_data['resources'] - new_resources = [ ResourceAssociation.objects.get(pk=a) - for a in new_resource_ids] + new_resources = [ResourceAssociation.objects.get(pk=a) + for a in new_resource_ids] first_session.resources = new_resources - if 'bethere' in form.changed_data and set(form.cleaned_data['bethere'])!=set(initial['bethere']): + if 'bethere' in form.changed_data and set(form.cleaned_data['bethere']) != set(initial['bethere']): first_session.constraints().filter(name='bethere').delete() bethere_cn = ConstraintName.objects.get(slug='bethere') for p in form.cleaned_data['bethere']: @@ -539,7 +765,7 @@ def edit(request, acronym, num=None): # deprecated # log activity - #add_session_activity(group,'Session Request was updated',meeting,user) + # add_session_activity(group,'Session Request was updated',meeting,user) # send notification if form.cleaned_data.get("send_notifications"): @@ -556,7 +782,7 @@ def edit(request, acronym, num=None): ) messages.success(request, 'Session Request updated') - return redirect('ietf.secr.sreq.views.view', acronym=acronym) + return redirect('ietf.meeting.views_session_request.view_request', acronym=acronym) else: # method is not POST # gather outbound conflicts for initial value @@ -567,142 +793,46 @@ def edit(request, acronym, num=None): initial['constraint_{}'.format(slug)] = ' '.join(groups) if not sessions: - return redirect('ietf.secr.sreq.views.new', acronym=acronym) - form = SessionForm(group, meeting, initial=initial, notifications_optional=has_role(request.user, "Secretariat")) + return redirect('ietf.meeting.views_session_request.new_request', acronym=acronym) + form = SessionRequestForm(group, meeting, initial=initial, notifications_optional=has_role(request.user, "Secretariat")) - return render(request, 'sreq/edit.html', { - 'is_locked': is_locked and not has_role(request.user,'Secretariat'), + return render(request, 'meeting/session_request_form.html', { + 'is_locked': is_locked and not has_role(request.user, 'Secretariat'), 'meeting': meeting, 'form': form, 'group': group, + 'is_create': False, 'session_conflicts': session_conflicts}, ) -@role_required(*AUTHORIZED_ROLES) -def main(request): - ''' - Display list of groups the user has access to. - - Template variables - form: a select box populated with unscheduled groups - meeting: the current meeting - scheduled_sessions: - ''' - # check for locked flag - is_locked = check_app_locked() - - if is_locked and not has_role(request.user,'Secretariat'): - message = get_lock_message() - return render(request, 'sreq/locked.html', { - 'message': message}, - ) - - meeting = get_meeting(days=14) - - scheduled_groups = [] - unscheduled_groups = [] - - group_types = GroupFeatures.objects.filter(has_meetings=True).values_list('type', flat=True) - - my_groups = [g for g in get_my_groups(request.user, conclude=True) if g.type_id in group_types] - - sessions_by_group = defaultdict(list) - for s in add_event_info_to_session_qs(Session.objects.filter(meeting=meeting, group__in=my_groups)).filter(current_status__in=['schedw', 'apprw', 'appr', 'sched']): - sessions_by_group[s.group_id].append(s) - - for group in my_groups: - group.meeting_sessions = sessions_by_group.get(group.pk, []) - - if group.pk in sessions_by_group: - # include even if concluded as we need to to see that the - # sessions are there - scheduled_groups.append(group) - else: - if group.state_id not in ['conclude', 'bof-conc']: - # too late for unscheduled if concluded - unscheduled_groups.append(group) - - # warn if there are no associated groups - if not scheduled_groups and not unscheduled_groups: - messages.warning(request, 'The account %s is not associated with any groups. If you have multiple Datatracker accounts you may try another or report a problem to %s' % (request.user, settings.SECRETARIAT_ACTION_EMAIL)) - - # add session status messages for use in template - for group in scheduled_groups: - if not group.features.acts_like_wg or (len(group.meeting_sessions) < 3): - group.status_message = group.meeting_sessions[0].current_status - else: - group.status_message = 'First two sessions: %s, Third session: %s' % (group.meeting_sessions[0].current_status, group.meeting_sessions[2].current_status) - - # add not meeting indicators for use in template - for group in unscheduled_groups: - if any(s.current_status == 'notmeet' for s in group.meeting_sessions): - group.not_meeting = True - - return render(request, 'sreq/main.html', { - 'is_locked': is_locked, - 'meeting': meeting, - 'scheduled_groups': scheduled_groups, - 'unscheduled_groups': unscheduled_groups}, - ) @check_permissions -def new(request, acronym): +def approve_request(request, acronym): ''' - This view gathers details for a new session request. The user proceeds to confirm() - to create the request. + This view approves the third session. For use by ADs or Secretariat. ''' - group = get_object_or_404(Group, acronym=acronym) - if len(group.features.session_purposes) == 0: - raise Http404(f'Cannot request sessions for group "{acronym}"') meeting = get_meeting(days=14) - session_conflicts = dict(inbound=inbound_session_conflicts_as_string(group, meeting)) - - # check if app is locked - is_locked = check_app_locked() - if is_locked and not has_role(request.user,'Secretariat'): - messages.warning(request, "The Session Request Tool is closed") - return redirect('ietf.secr.sreq.views.main') - - if request.method == 'POST': - button_text = request.POST.get('submit', '') - if button_text == 'Cancel': - return redirect('ietf.secr.sreq.views.main') - - form = SessionForm(group, meeting, request.POST, notifications_optional=has_role(request.user, "Secretariat")) - if form.is_valid(): - return confirm(request, acronym) + group = get_object_or_404(Group, acronym=acronym) - # the "previous" querystring causes the form to be returned - # pre-populated with data from last meeeting's session request - elif request.method == 'GET' and 'previous' in request.GET: - latest_session = add_event_info_to_session_qs(Session.objects.filter(meeting__type_id='ietf', group=group)).exclude(current_status__in=['notmeet', 'deleted', 'canceled',]).order_by('-meeting__date').first() - if latest_session: - previous_meeting = Meeting.objects.get(number=latest_session.meeting.number) - previous_sessions = add_event_info_to_session_qs(Session.objects.filter(meeting=previous_meeting, group=group)).exclude(current_status__in=['notmeet', 'deleted']).order_by('id') - if not previous_sessions: - messages.warning(request, 'This group did not meet at %s' % previous_meeting) - return redirect('ietf.secr.sreq.views.new', acronym=acronym) - else: - messages.info(request, 'Fetched session info from %s' % previous_meeting) - else: - messages.warning(request, 'Did not find any previous meeting') - return redirect('ietf.secr.sreq.views.new', acronym=acronym) + session = add_event_info_to_session_qs(Session.objects.filter(meeting=meeting, group=group)).filter(current_status='apprw').first() + if session is None: + raise Http404 - initial = get_initial_session(previous_sessions, prune_conflicts=True) - if 'resources' in initial: - initial['resources'] = [x.pk for x in initial['resources']] - form = SessionForm(group, meeting, initial=initial, notifications_optional=has_role(request.user, "Secretariat")) + if has_role(request.user, 'Secretariat') or group.parent.role_set.filter(name='ad', person=request.user.person): + SchedulingEvent.objects.create( + session=session, + status=SessionStatusName.objects.get(slug='appr'), + by=request.user.person, + ) + session_changed(session) + messages.success(request, 'Third session approved') + return redirect('ietf.meeting.views_session_request.view_request', acronym=acronym) else: - initial={} - form = SessionForm(group, meeting, initial=initial, notifications_optional=has_role(request.user, "Secretariat")) + # if an unauthorized user gets here return error + messages.error(request, 'Not authorized to approve the third session') + return redirect('ietf.meeting.views_session_request.view_request', acronym=acronym) - return render(request, 'sreq/new.html', { - 'meeting': meeting, - 'form': form, - 'group': group, - 'session_conflicts': session_conflicts}, - ) @check_permissions def no_session(request, acronym): @@ -722,7 +852,7 @@ def no_session(request, acronym): # skip if state is already notmeet if add_event_info_to_session_qs(Session.objects.filter(group=group, meeting=meeting)).filter(current_status='notmeet'): messages.info(request, 'The group %s is already marked as not meeting' % group.acronym) - return redirect('ietf.secr.sreq.views.main') + return redirect('ietf.meeting.views_session_request.list_view') session = Session.objects.create( group=group, @@ -740,125 +870,62 @@ def no_session(request, acronym): session_changed(session) # send notification - (to_email, cc_list) = gather_address_lists('session_request_not_meeting',group=group,person=login) + (to_email, cc_list) = gather_address_lists('session_request_not_meeting', group=group, person=login) from_email = (settings.SESSION_REQUEST_FROM_EMAIL) subject = '%s - Not having a session at IETF %s' % (group.acronym, meeting.number) - send_mail(request, to_email, from_email, subject, 'sreq/not_meeting_notification.txt', - {'login':login, - 'group':group, - 'meeting':meeting}, cc=cc_list) + send_mail(request, to_email, from_email, subject, 'meeting/session_not_meeting_notification.txt', + {'login': login, + 'group': group, + 'meeting': meeting}, cc=cc_list) # deprecated? # log activity - #text = 'A message was sent to notify not having a session at IETF %d' % meeting.meeting_num - #add_session_activity(group,text,meeting,request.person) + # text = 'A message was sent to notify not having a session at IETF %d' % meeting.meeting_num + # add_session_activity(group,text,meeting,request.person) # redirect messages.success(request, 'A message was sent to notify not having a session at IETF %s' % meeting.number) - return redirect('ietf.secr.sreq.views.main') - -@role_required('Secretariat') -def tool_status(request): - ''' - This view handles locking and unlocking of the tool to the public. - ''' - meeting = get_meeting(days=14) - is_locked = check_app_locked(meeting=meeting) - - if request.method == 'POST': - button_text = request.POST.get('submit', '') - if button_text == 'Back': - return redirect('ietf.secr.sreq.views.main') - - form = ToolStatusForm(request.POST) - - if button_text == 'Lock': - if form.is_valid(): - meeting.session_request_lock_message = form.cleaned_data['message'] - meeting.save() - messages.success(request, 'Session Request Tool is now Locked') - return redirect('ietf.secr.sreq.views.main') - - elif button_text == 'Unlock': - meeting.session_request_lock_message = '' - meeting.save() - messages.success(request, 'Session Request Tool is now Unlocked') - return redirect('ietf.secr.sreq.views.main') - - else: - if is_locked: - message = get_lock_message() - initial = {'message': message} - form = ToolStatusForm(initial=initial) - else: - form = ToolStatusForm() + return redirect('ietf.meeting.views_session_request.list_view') - return render(request, 'sreq/tool_status.html', { - 'is_locked': is_locked, - 'form': form}, - ) -@role_required(*AUTHORIZED_ROLES) -def view(request, acronym, num = None): +@check_permissions +def cancel_request(request, acronym): ''' - This view displays the session request info + This view cancels a session request and sends a notification. + To cancel, or withdraw the request set status = deleted. + "canceled" status is used by the secretariat. + + NOTE: this function can also be called after a session has been + scheduled during the period when the session request tool is + reopened. In this case be sure to clear the timeslot assignment as well. ''' - meeting = get_meeting(num,days=14) + meeting = get_meeting(days=14) group = get_object_or_404(Group, acronym=acronym) - sessions = add_event_info_to_session_qs(Session.objects.filter(meeting=meeting, group=group)).filter(Q(current_status__isnull=True) | ~Q(current_status__in=('canceled','notmeet','deleted'))).order_by('id') - - # check if app is locked - is_locked = check_app_locked() - if is_locked: - messages.warning(request, "The Session Request Tool is closed") - - # if there are no session requests yet, redirect to new session request page - if not sessions: - if is_locked: - return redirect('ietf.secr.sreq.views.main') - else: - return redirect('ietf.secr.sreq.views.new', acronym=acronym) - - activities = [{ - 'act_date': e.time.strftime('%b %d, %Y'), - 'act_time': e.time.strftime('%H:%M:%S'), - 'activity': e.status.name, - 'act_by': e.by, - } for e in sessions[0].schedulingevent_set.select_related('status', 'by')] - - # gather outbound conflicts - outbound_dict = OrderedDict() - for obc in group.constraint_source_set.filter(meeting=meeting, name__is_group_conflict=True): - if obc.name.slug not in outbound_dict: - outbound_dict[obc.name.slug] = [] - outbound_dict[obc.name.slug].append(obc.target.acronym) - - session_conflicts = dict( - inbound=inbound_session_conflicts_as_string(group, meeting), - outbound=[dict(name=ConstraintName.objects.get(slug=slug), groups=' '.join(groups)) - for slug, groups in outbound_dict.items()], - ) + sessions = Session.objects.filter(meeting=meeting, group=group).order_by('id') + login = request.user.person - show_approve_button = False + # delete conflicts + Constraint.objects.filter(meeting=meeting, source=group).delete() - # if sessions include a 3rd session waiting approval and the user is a secretariat or AD of the group - # display approve button - if any(s.current_status == 'apprw' for s in sessions): - if has_role(request.user,'Secretariat') or group.parent.role_set.filter(name='ad',person=request.user.person): - show_approve_button = True + # mark sessions as deleted + for session in sessions: + SchedulingEvent.objects.create( + session=session, + status=SessionStatusName.objects.get(slug='deleted'), + by=request.user.person, + ) + session_changed(session) - # build session dictionary (like querydict from new session request form) for use in template - session = get_initial_session(sessions) + # clear schedule assignments if already scheduled + session.timeslotassignments.all().delete() - return render(request, 'sreq/view.html', { - 'can_edit': (not is_locked) or has_role(request.user, 'Secretariat'), - 'can_cancel': (not is_locked) or has_role(request.user, 'Secretariat'), - 'session': session, # legacy processed data - 'sessions': sessions, # actual session instances - 'activities': activities, - 'meeting': meeting, - 'group': group, - 'session_conflicts': session_conflicts, - 'show_approve_button': show_approve_button}, - ) + # send notifitcation + (to_email, cc_list) = gather_address_lists('session_request_cancelled', group=group, person=login) + from_email = (settings.SESSION_REQUEST_FROM_EMAIL) + subject = '%s - Cancelling a meeting request for IETF %s' % (group.acronym, meeting.number) + send_mail(request, to_email, from_email, subject, 'meeting/session_cancel_notification.txt', + {'requester': get_requester_text(login, group), + 'meeting': meeting}, cc=cc_list) + messages.success(request, 'The %s Session Request has been cancelled' % group.acronym) + return redirect('ietf.meeting.views_session_request.list_view') diff --git a/ietf/message/admin.py b/ietf/message/admin.py index 250e1eb596..6a876cdc70 100644 --- a/ietf/message/admin.py +++ b/ietf/message/admin.py @@ -27,7 +27,8 @@ def queryset(self, request, queryset): class MessageAdmin(admin.ModelAdmin): - list_display = ["sent_status", "subject", "by", "time", "groups"] + list_display = ["sent_status", "display_subject", "by", "time", "groups"] + list_display_links = ["display_subject"] search_fields = ["subject", "body"] raw_id_fields = ["by", "related_groups", "related_docs"] list_filter = [ @@ -37,6 +38,10 @@ class MessageAdmin(admin.ModelAdmin): ordering = ["-time"] actions = ["retry_send"] + @admin.display(description="Subject", empty_value="(no subject)") + def display_subject(self, instance): + return instance.subject or None # None triggers the empty_value + def groups(self, instance): return ", ".join(g.acronym for g in instance.related_groups.all()) diff --git a/ietf/middleware.py b/ietf/middleware.py index a4b7a0d24c..fa2e8efd0c 100644 --- a/ietf/middleware.py +++ b/ietf/middleware.py @@ -8,6 +8,7 @@ from django.http import HttpResponsePermanentRedirect from ietf.utils.log import log, exc_parts from ietf.utils.mail import log_smtp_exception +from opentelemetry.propagate import inject import re import smtplib import unicodedata @@ -99,3 +100,12 @@ def add_header(request): return response return add_header + +def add_otel_traceparent_header(get_response): + """Middleware to add the OpenTelemetry traceparent id header to the response""" + def add_header(request): + response = get_response(request) + inject(response) + return response + + return add_header diff --git a/ietf/name/admin.py b/ietf/name/admin.py index 4336e0569c..b89d6d141c 100644 --- a/ietf/name/admin.py +++ b/ietf/name/admin.py @@ -57,6 +57,7 @@ from ietf.stats.models import CountryAlias +from ietf.utils.admin import SaferTabularInline class NameAdmin(admin.ModelAdmin): @@ -86,7 +87,7 @@ class GroupTypeNameAdmin(NameAdmin): admin.site.register(GroupTypeName, GroupTypeNameAdmin) -class CountryAliasInline(admin.TabularInline): +class CountryAliasInline(SaferTabularInline): model = CountryAlias extra = 1 diff --git a/ietf/name/fixtures/names.json b/ietf/name/fixtures/names.json index c94e15a459..64e26e503a 100644 --- a/ietf/name/fixtures/names.json +++ b/ietf/name/fixtures/names.json @@ -650,7 +650,7 @@ }, { "fields": { - "desc": "4.2.1. Call for Adoption by WG Issued\r\n\r\n The \"Call for Adoption by WG Issued\" state should be used to indicate when an I-D is being considered for adoption by an IETF WG. An I-D that is in this state is actively being considered for adoption and has not yet achieved consensus, preference, or selection in the WG.\r\n\r\n This state may be used to describe an I-D that someone has asked a WG to consider for adoption, if the WG Chair has agreed with the request. This state may also be used to identify an I-D that a WG Chair asked an author to write specifically for consideration as a candidate WG item [WGDTSPEC], and/or an I-D that is listed as a 'candidate draft' in the WG's charter.\r\n\r\n Under normal conditions, it should not be possible for an I-D to be in the \"Call for Adoption by WG Issued\" state in more than one working group at the same time. This said, it is not uncommon for authors to \"shop\" their I-Ds to more than one WG at a time, with the hope of getting their documents adopted somewhere.\r\n\r\n After this state is implemented in the Datatracker, an I-D that is in the \"Call for Adoption by WG Issued\" state will not be able to be \"shopped\" to any other WG without the consent of the WG Chairs and the responsible ADs impacted by the shopping.\r\n\r\n Note that Figure 1 includes an arc leading from this state to outside of the WG state machine. This illustrates that some I-Ds that are considered do not get adopted as WG drafts. An I-D that is not adopted as a WG draft will transition out of the WG state machine and revert back to having no stream-specific state; however, the status change history log of the I-D will record that the I-D was previously in the \"Call for Adoption by WG Issued\" state.", + "desc": "A call for adoption of the individual submission document has been issued by the Working Group (WG) chairs. This call is still running but the WG has not yet reached consensus for adoption.", "name": "Call For Adoption By WG Issued", "next_states": [ 36, @@ -666,7 +666,7 @@ }, { "fields": { - "desc": "4.2.2. Adopted by a WG\r\n\r\n The \"Adopted by a WG\" state describes an individual submission I-D that an IETF WG has agreed to adopt as one of its WG drafts.\r\n\r\n WG Chairs who use this state will be able to clearly indicate when their WGs adopt individual submission I-Ds. This will facilitate the Datatracker's ability to correctly capture \"Replaces\" information for WG drafts and correct \"Replaced by\" information for individual submission I-Ds that have been replaced by WG drafts.\r\n\r\n This state is needed because the Datatracker uses the filename of an I-D as a key to search its database for status information about the I-D, and because the filename of a WG I-D is supposed to be different from the filename of an individual submission I-D. The filename of an individual submission I-D will typically be formatted as 'draft-author-wgname-topic-nn'.\r\n\r\n The filename of a WG document is supposed to be formatted as 'draft- ietf-wgname-topic-nn'.\r\n\r\n An individual I-D that is adopted by a WG may take weeks or months to be resubmitted by the author as a new (version-00) WG draft. If the \"Adopted by a WG\" state is not used, the Datatracker has no way to determine that an I-D has been adopted until a new version of the I-D is submitted to the WG by the author and until the I-D is approved for posting by a WG Chair.", + "desc": "The individual submission document has been adopted by the Working Group (WG), but a WG document replacing this document with the typical naming convention of 'draft- ietf-wgname-topic-nn' has not yet been submitted.", "name": "Adopted by a WG", "next_states": [ 38 @@ -681,7 +681,7 @@ }, { "fields": { - "desc": "4.2.3. Adopted for WG Info Only\r\n\r\n The \"Adopted for WG Info Only\" state describes a document that contains useful information for the WG that adopted it, but the document is not intended to be published as an RFC. The WG will not actively develop the contents of the I-D or progress it for publication as an RFC. The only purpose of the I-D is to provide information for internal use by the WG.", + "desc": "The document is adopted by the Working Group (WG) for its internal use. The WG has decided that it will not pursue publication of it as an RFC.", "name": "Adopted for WG Info Only", "next_states": [], "order": 3, @@ -694,7 +694,7 @@ }, { "fields": { - "desc": "4.2.4. WG Document\r\n\r\n The \"WG Document\" state describes an I-D that has been adopted by an IETF WG and is being actively developed.\r\n\r\n A WG Chair may transition an I-D into the \"WG Document\" state at any time as long as the I-D is not being considered or developed in any other WG.\r\n\r\n Alternatively, WG Chairs may rely upon new functionality to be added to the Datatracker to automatically move version-00 drafts into the \"WG Document\" state as described in Section 4.1.\r\n\r\n Under normal conditions, it should not be possible for an I-D to be in the \"WG Document\" state in more than one WG at a time. This said, I-Ds may be transferred from one WG to another with the consent of the WG Chairs and the responsible ADs.", + "desc": "The document has been adopted by the Working Group (WG) and is under development. A document can only be adopted by one WG at a time. However, a document may be transferred between WGs.", "name": "WG Document", "next_states": [ 39, @@ -712,7 +712,7 @@ }, { "fields": { - "desc": "4.2.5. Parked WG Document\r\n\r\n A \"Parked WG Document\" is an I-D that has lost its author or editor, is waiting for another document to be written or for a review to be completed, or cannot be progressed by the working group for some other reason.\r\n\r\n Some of the annotation tags described in Section 4.3 may be used in conjunction with this state to indicate why an I-D has been parked, and/or what may need to happen for the I-D to be un-parked.\r\n\r\n Parking a WG draft will not prevent it from expiring; however, this state can be used to indicate why the I-D has stopped progressing in the WG.\r\n\r\n A \"Parked WG Document\" that is not expired may be transferred from one WG to another with the consent of the WG Chairs and the responsible ADs.", + "desc": "The Working Group (WG) document is in a temporary state where it will not be actively developed. The reason for the pause is explained via a datatracker comments section.", "name": "Parked WG Document", "next_states": [ 38 @@ -727,7 +727,7 @@ }, { "fields": { - "desc": "4.2.6. Dead WG Document\r\n\r\n A \"Dead WG Document\" is an I-D that has been abandoned. Note that 'Dead' is not always a final state for a WG I-D. If consensus is subsequently achieved, a \"Dead WG Document\" may be resurrected. A \"Dead WG Document\" that is not resurrected will eventually expire.\r\n\r\n Note that an I-D that is declared to be \"Dead\" in one WG and that is not expired may be transferred to a non-dead state in another WG with the consent of the WG Chairs and the responsible ADs.", + "desc": "The Working Group (WG) document has been abandoned by the WG. No further development is planned in this WG. A decision to resume work on this document and move it out of this state is possible.", "name": "Dead WG Document", "next_states": [ 38 @@ -742,7 +742,7 @@ }, { "fields": { - "desc": "4.2.7. In WG Last Call\r\n\r\n A document \"In WG Last Call\" is an I-D for which a WG Last Call (WGLC) has been issued and is in progress.\r\n\r\n Note that conducting a WGLC is an optional part of the IETF WG process, per Section 7.4 of RFC 2418 [RFC2418].\r\n\r\n If a WG Chair decides to conduct a WGLC on an I-D, the \"In WG Last Call\" state can be used to track the progress of the WGLC. The Chair may configure the Datatracker to send a WGLC message to one or more mailing lists when the Chair moves the I-D into this state. The WG Chair may also be able to select a different set of mailing lists for a different document undergoing a WGLC; some documents may deserve coordination with other WGs.\r\n\r\n A WG I-D in this state should remain \"In WG Last Call\" until the WG Chair moves it to another state. The WG Chair may configure the Datatracker to send an e-mail after a specified period of time to remind or 'nudge' the Chair to conclude the WGLC and to determine the next state for the document.\r\n\r\n It is possible for one WGLC to lead into another WGLC for the same document. For example, an I-D that completed a WGLC as an \"Informational\" document may need another WGLC if a decision is taken to convert the I-D into a Standards Track document.", + "desc": "The Working Group (WG) document is currently subject to an active WG Last Call (WGLC) review per Section 7.4 of RFC2418.", "name": "In WG Last Call", "next_states": [ 38, @@ -759,7 +759,7 @@ }, { "fields": { - "desc": "4.2.8. Waiting for WG Chair Go-Ahead\r\n\r\n A WG Chair may wish to place an I-D that receives a lot of comments during a WGLC into the \"Waiting for WG Chair Go-Ahead\" state. This state describes an I-D that has undergone a WGLC; however, the Chair is not yet ready to call consensus on the document.\r\n\r\n If comments from the WGLC need to be responded to, or a revision to the I-D is needed, the Chair may place an I-D into this state until all of the WGLC comments are adequately addressed and the (possibly revised) document is in the I-D repository.", + "desc": "The Working Group (WG) document has completed Working Group Last Call (WGLC), but the WG chair(s) are not yet ready to call consensus on the document. The reasons for this may include comments from the WGLC need to be responded to, or a revision to the document is needed", "name": "Waiting for WG Chair Go-Ahead", "next_states": [ 41, @@ -775,7 +775,7 @@ }, { "fields": { - "desc": "4.2.9. WG Consensus: Waiting for Writeup\r\n\r\n A document in the \"WG Consensus: Waiting for Writeup\" state has essentially completed its development within the working group, and is nearly ready to be sent to the IESG for publication. The last thing to be done is the preparation of a protocol writeup by a Document Shepherd. The IESG requires that a document shepherd writeup be completed before publication of the I-D is requested. The IETF document shepherding process and the role of a WG Document Shepherd is described in RFC 4858 [RFC4858]\r\n\r\n A WG Chair may call consensus on an I-D without a formal WGLC and transition an I-D that was in the \"WG Document\" state directly into this state.\r\n\r\n The name of this state includes the words \"Waiting for Writeup\" because a good document shepherd writeup takes time to prepare.", + "desc": "The Working Group (WG) document has consensus to proceed to publication. However, the document is waiting for a document shepherd write-up per RFC4858.", "name": "WG Consensus: Waiting for Write-Up", "next_states": [ 44 @@ -790,7 +790,7 @@ }, { "fields": { - "desc": "4.2.10. Submitted to IESG for Publication\r\n\r\n This state describes a WG document that has been submitted to the IESG for publication and that has not been sent back to the working group for revision.\r\n\r\n An I-D in this state may be under review by the IESG, it may have been approved and be in the RFC Editor's queue, or it may have been published as an RFC. Other possibilities exist too. The document may be \"Dead\" (in the IESG state machine) or in a \"Do Not Publish\" state.", + "desc": "The Working Group (WG) document has left the WG and been submitted to the Internet Engineering Steering Group (IESG) for evaluation and publication. See the “IESG State” or “RFC Editor State” for further details on the state of the document.", "name": "Submitted to IESG for Publication", "next_states": [ 38 @@ -2020,7 +2020,7 @@ }, { "fields": { - "desc": "The document has been marked as a candidate for WG adoption by the WG Chair. This state can be used before a call for adoption is issued (and the document is put in the \"Call For Adoption By WG Issued\" state), to indicate that the document is in the queue for a call for adoption, even if none has been issued yet.", + "desc": "The individual submission document has been marked by the Working Group (WG) chairs as a candidate for adoption by the WG, but no adoption call has been started.", "name": "Candidate for WG Adoption", "next_states": [ 35 @@ -2152,7 +2152,7 @@ }, { "fields": { - "desc": "In some areas, it can be desirable to wait for multiple interoperable implementations before progressing a draft to be an RFC, and in some WGs this is required. This state should be entered after WG Last Call has completed.", + "desc": "The progression of this Working Group (WG) document towards publication is paused as it awaits implementation. The process governing the approach to implementations is WG-specific.", "name": "Waiting for Implementation", "next_states": [], "order": 8, @@ -2165,7 +2165,7 @@ }, { "fields": { - "desc": "Held by WG, see document history for details.", + "desc": "Held by Working Group (WG) chairs for administrative reasons. See document history for details.", "name": "Held by WG", "next_states": [], "order": 9, @@ -2638,11 +2638,24 @@ "order": 0, "slug": "historic", "type": "statement", - "used": true + "used": false }, "model": "doc.state", "pk": 182 }, + { + "fields": { + "desc": "The statement is no longer active", + "name": "Inactive", + "next_states": [], + "order": 0, + "slug": "inactive", + "type": "statement", + "used": true + }, + "model": "doc.state", + "pk": 183 + }, { "fields": { "label": "State" @@ -4460,6 +4473,7 @@ ], "session_purposes": [ "coding", + "open_meeting", "presentation", "social", "tutorial" @@ -5520,13 +5534,30 @@ "liaison_response_contacts", "liaison_technical_contacts" ], - "desc": "Recipient for a message when a new liaison statement is posted", + "desc": "Recipients for a message when a new incoming liaison statement is posted", + "to": [ + "liaison_to_contacts" + ] + }, + "model": "mailtrigger.mailtrigger", + "pk": "liaison_statement_posted_incoming" + }, + { + "fields": { + "cc": [ + "liaison_cc", + "liaison_coordinators", + "liaison_from_contact", + "liaison_response_contacts", + "liaison_technical_contacts" + ], + "desc": "Recipients for a message when a new outgoing liaison statement is posted", "to": [ "liaison_to_contacts" ] }, "model": "mailtrigger.mailtrigger", - "pk": "liaison_statement_posted" + "pk": "liaison_statement_posted_outgoing" }, { "fields": { @@ -7068,6 +7099,14 @@ "model": "mailtrigger.recipient", "pk": "liaison_coordinators" }, + { + "fields": { + "desc": "Email address of the formal sender of the statement", + "template": "{{liaison.from_contact}}" + }, + "model": "mailtrigger.recipient", + "pk": "liaison_from_contact" + }, { "fields": { "desc": "The assigned liaison manager for an external group ", diff --git a/ietf/name/serializers.py b/ietf/name/serializers.py new file mode 100644 index 0000000000..a764f56051 --- /dev/null +++ b/ietf/name/serializers.py @@ -0,0 +1,11 @@ +# Copyright The IETF Trust 2024, All Rights Reserved +"""django-rest-framework serializers""" +from rest_framework import serializers + +from .models import StreamName + + +class StreamNameSerializer(serializers.ModelSerializer): + class Meta: + model = StreamName + fields = ["slug", "name", "desc"] diff --git a/ietf/nomcom/tests.py b/ietf/nomcom/tests.py index cc2e0826d3..210788ce07 100644 --- a/ietf/nomcom/tests.py +++ b/ietf/nomcom/tests.py @@ -1,5 +1,4 @@ -# Copyright The IETF Trust 2012-2023, All Rights Reserved -# -*- coding: utf-8 -*- +# Copyright The IETF Trust 2012-2025, All Rights Reserved import datetime @@ -27,8 +26,14 @@ from ietf.api.views import EmailIngestionError from ietf.dbtemplate.factories import DBTemplateFactory from ietf.dbtemplate.models import DBTemplate -from ietf.doc.factories import DocEventFactory, WgDocumentAuthorFactory, \ - NewRevisionDocEventFactory, DocumentAuthorFactory +from ietf.doc.factories import ( + DocEventFactory, + WgDocumentAuthorFactory, + NewRevisionDocEventFactory, + DocumentAuthorFactory, + RfcAuthorFactory, + WgDraftFactory, WgRfcFactory, +) from ietf.group.factories import GroupFactory, GroupHistoryFactory, RoleFactory, RoleHistoryFactory from ietf.group.models import Group, Role from ietf.meeting.factories import MeetingFactory, AttendedFactory, RegistrationFactory @@ -45,10 +50,20 @@ nomcom_kwargs_for_year, provide_private_key_to_test_client, \ key from ietf.nomcom.tasks import send_nomcom_reminders_task -from ietf.nomcom.utils import get_nomcom_by_year, make_nomineeposition, \ - get_hash_nominee_position, is_eligible, list_eligible, \ - get_eligibility_date, suggest_affiliation, ingest_feedback_email, \ - decorate_volunteers_with_qualifications, send_reminders, _is_time_to_send_reminder +from ietf.nomcom.utils import ( + get_nomcom_by_year, + make_nomineeposition, + get_hash_nominee_position, + is_eligible, + list_eligible, + get_eligibility_date, + suggest_affiliation, + ingest_feedback_email, + decorate_volunteers_with_qualifications, + send_reminders, + _is_time_to_send_reminder, + get_qualified_author_queryset, +) from ietf.person.factories import PersonFactory, EmailFactory from ietf.person.models import Email, Person from ietf.utils.mail import outbox, empty_outbox, get_payload_text @@ -2440,6 +2455,85 @@ def test_get_eligibility_date(self): NomComFactory(group__acronym=f'nomcom{this_year}', first_call_for_volunteers=datetime.date(this_year,5,6)) self.assertEqual(get_eligibility_date(),datetime.date(this_year,5,6)) + def test_get_qualified_author_queryset(self): + """get_qualified_author_queryset implements the eligiblity rules correctly + + This is not an exhaustive test of corner cases. Overlaps considerably with + rfc8989EligibilityTests.test_elig_by_author(). + """ + people = PersonFactory.create_batch(2) + extra_person = PersonFactory() + base_qs = Person.objects.filter(pk__in=[person.pk for person in people]) + now = datetime.datetime.now(tz=datetime.UTC) + one_year = datetime.timedelta(days=365) + + # Authors with no qualifying drafts + self.assertCountEqual( + get_qualified_author_queryset(base_qs, now - 5 * one_year, now), [] + ) + + # Authors with one qualifying draft + approved_draft = WgDraftFactory(authors=people, states=[("draft", "active")]) + DocEventFactory( + type="iesg_approved", + doc=approved_draft, + time=now - 4 * one_year, + ) + self.assertCountEqual( + get_qualified_author_queryset(base_qs, now - 5 * one_year, now), [] + ) + + # Create a draft that was published into an RFC. Give it an extra author who + # should not be eligible. + published_draft = WgDraftFactory(authors=people, states=[("draft", "rfc")]) + DocEventFactory( + type="iesg_approved", + doc=published_draft, + time=now - 5.5 * one_year, # < 6 years ago + ) + rfc = WgRfcFactory( + authors=people + [extra_person], + group=published_draft.group, + ) + DocEventFactory( + type="published_rfc", + doc=rfc, + time=now - 0.5 * one_year, # < 1 year ago + ) + # Period 6 years ago to 1 year ago - authors are eligible due to the + # iesg-approved draft in this window + self.assertCountEqual( + get_qualified_author_queryset(base_qs, now - 6 * one_year, now - one_year), + people, + ) + + # Period 5 years ago to now - authors are eligible due to the RFC publication + self.assertCountEqual( + get_qualified_author_queryset(base_qs, now - 5 * one_year, now), + people, + ) + + # Use the extra_person to check that a single doc can't count both as an + # RFC _and_ an approved draft. Use an eligibility interval that includes both + # the approval and the RFC publication + self.assertCountEqual( + get_qualified_author_queryset(base_qs, now - 6 * one_year, now), + people, # does not include extra_person! + ) + + # Now add an RfcAuthor for only one of the two authors to the RFC. This should + # remove the other author from the eligibility list because the DocumentAuthor + # records are no longer used. + RfcAuthorFactory( + document=rfc, + person=people[0], + titlepage_name="P. Zero", + ) + self.assertCountEqual( + get_qualified_author_queryset(base_qs, now - 5 * one_year, now), + [people[0]], + ) + class rfc8713EligibilityTests(TestCase): @@ -2724,33 +2818,41 @@ def test_elig_by_author(self): ineligible = set() p = PersonFactory() - ineligible.add(p) - + ineligible.add(p) # no RFCs or iesg-approved drafts p = PersonFactory() - da = WgDocumentAuthorFactory(person=p) - DocEventFactory(type='published_rfc',doc=da.document,time=middle_date) - ineligible.add(p) + doc = WgRfcFactory(authors=[p]) + DocEventFactory(type='published_rfc', doc=doc, time=middle_date) + ineligible.add(p) # only one RFC p = PersonFactory() - da = WgDocumentAuthorFactory(person=p) + da = WgDocumentAuthorFactory( + person=p, + document__states=[("draft", "active"), ("draft-rfceditor", "ref")], + ) DocEventFactory(type='iesg_approved',doc=da.document,time=last_date) - da = WgDocumentAuthorFactory(person=p) - DocEventFactory(type='published_rfc',doc=da.document,time=first_date) - eligible.add(p) + doc = WgRfcFactory(authors=[p]) + DocEventFactory(type='published_rfc', doc=doc, time=first_date) + eligible.add(p) # one RFC and one iesg-approved draft p = PersonFactory() - da = WgDocumentAuthorFactory(person=p) + da = WgDocumentAuthorFactory( + person=p, + document__states=[("draft", "active"), ("draft-rfceditor", "ref")], + ) DocEventFactory(type='iesg_approved',doc=da.document,time=middle_date) - da = WgDocumentAuthorFactory(person=p) - DocEventFactory(type='published_rfc',doc=da.document,time=day_before_first_date) - ineligible.add(p) + doc = WgRfcFactory(authors=[p]) + DocEventFactory(type='published_rfc', doc=doc, time=day_before_first_date) + ineligible.add(p) # RFC is out of the eligibility window p = PersonFactory() - da = WgDocumentAuthorFactory(person=p) + da = WgDocumentAuthorFactory( + person=p, + document__states=[("draft", "active"), ("draft-rfceditor", "ref")], + ) DocEventFactory(type='iesg_approved',doc=da.document,time=day_after_last_date) - da = WgDocumentAuthorFactory(person=p) - DocEventFactory(type='published_rfc',doc=da.document,time=middle_date) - ineligible.add(p) + doc = WgRfcFactory(authors=[p]) + DocEventFactory(type='published_rfc', doc=doc, time=middle_date) + ineligible.add(p) # iesg approval is outside the eligibility window for person in eligible: self.assertTrue(is_eligible(person,nomcom)) @@ -2878,15 +2980,38 @@ def test_volunteer(self): def test_suggest_affiliation(self): person = PersonFactory() - self.assertEqual(suggest_affiliation(person), '') - da = DocumentAuthorFactory(person=person,affiliation='auth_affil') + self.assertEqual(suggest_affiliation(person), "") + rfc_da = DocumentAuthorFactory( + person=person, + document__type_id="rfc", + affiliation="", + ) + rfc = rfc_da.document + DocEventFactory(doc=rfc, type="published_rfc") + self.assertEqual(suggest_affiliation(person), "") + + rfc_da.affiliation = "rfc_da_affil" + rfc_da.save() + self.assertEqual(suggest_affiliation(person), "rfc_da_affil") + + rfc_ra = RfcAuthorFactory(person=person, document=rfc, affiliation="") + self.assertEqual(suggest_affiliation(person), "") + + rfc_ra.affiliation = "rfc_ra_affil" + rfc_ra.save() + self.assertEqual(suggest_affiliation(person), "rfc_ra_affil") + + da = DocumentAuthorFactory(person=person, affiliation="auth_affil") NewRevisionDocEventFactory(doc=da.document) - self.assertEqual(suggest_affiliation(person), 'auth_affil') + self.assertEqual(suggest_affiliation(person), "auth_affil") + nc = NomComFactory() - nc.volunteer_set.create(person=person,affiliation='volunteer_affil') - self.assertEqual(suggest_affiliation(person), 'volunteer_affil') - RegistrationFactory(person=person, affiliation='meeting_affil') - self.assertEqual(suggest_affiliation(person), 'meeting_affil') + nc.volunteer_set.create(person=person, affiliation="volunteer_affil") + self.assertEqual(suggest_affiliation(person), "volunteer_affil") + + RegistrationFactory(person=person, affiliation="meeting_affil") + self.assertEqual(suggest_affiliation(person), "meeting_affil") + class VolunteerDecoratorUnitTests(TestCase): def test_decorate_volunteers_with_qualifications(self): @@ -2922,15 +3047,15 @@ def test_decorate_volunteers_with_qualifications(self): author_person = PersonFactory() for i in range(2): - da = WgDocumentAuthorFactory(person=author_person) + doc = WgRfcFactory(authors=[author_person]) DocEventFactory( type='published_rfc', - doc=da.document, + doc=doc, time=datetime.datetime( elig_date.year - 3, elig_date.month, 28 if elig_date.month == 2 and elig_date.day == 29 else elig_date.day, - tzinfo=datetime.timezone.utc, + tzinfo=datetime.UTC, ) ) nomcom.volunteer_set.create(person=author_person) diff --git a/ietf/nomcom/utils.py b/ietf/nomcom/utils.py index dd651c2941..a2ab680df6 100644 --- a/ietf/nomcom/utils.py +++ b/ietf/nomcom/utils.py @@ -18,7 +18,7 @@ from email.utils import parseaddr from textwrap import dedent -from django.db.models import Q, Count +from django.db.models import Q, Count, F, QuerySet from django.conf import settings from django.contrib.sites.models import Site from django.core.exceptions import ObjectDoesNotExist @@ -27,7 +27,7 @@ from django.shortcuts import get_object_or_404 from ietf.dbtemplate.models import DBTemplate -from ietf.doc.models import DocEvent, NewRevisionDocEvent +from ietf.doc.models import DocEvent, NewRevisionDocEvent, Document from ietf.group.models import Group, Role from ietf.person.models import Email, Person from ietf.mailtrigger.utils import gather_address_lists @@ -576,6 +576,70 @@ def get_8989_eligibility_querysets(date, base_qs): def get_9389_eligibility_querysets(date, base_qs): return get_threerule_eligibility_querysets(date, base_qs, three_of_five_callable=three_of_five_eligible_9389) + +def get_qualified_author_queryset( + base_qs: QuerySet[Person], + eligibility_period_start: datetime.datetime, + eligibility_period_end: datetime.datetime, +): + """Filter a Person queryset, keeping those qualified by RFC 8989's author path + + The author path is defined by "path 3" in section 4 of RFC 8989. It qualifies + a person who has been a front-page listed author or editor of at least two IETF- + stream RFCs within the last five years. An I-D in the RFC Editor queue that was + approved by the IESG is treated as an RFC, using the date of entry to the RFC + Editor queue as the date for qualification. + + This method does not strictly enforce "in the RFC Editor queue" for IESG-approved + drafts when computing eligibility. In the overwhelming majority of cases, an IESG- + approved draft immediately enters the queue and goes on to be published, so this + simplification makes the calculation much easier and virtually never affects + eligibility. + + Arguments eligibility_period_start and eligibility_period_end are datetimes that + mark the start and end of the eligibility period. These should be five years apart. + """ + # First, get the RFCs using publication date + qualifying_rfc_pub_events = DocEvent.objects.filter( + type='published_rfc', + time__gte=eligibility_period_start, + time__lte=eligibility_period_end, + ) + qualifying_rfcs = Document.objects.filter( + type_id="rfc", + docevent__in=qualifying_rfc_pub_events + ).annotate( + rfcauthor_count=Count("rfcauthor") + ) + rfcs_with_rfcauthors = qualifying_rfcs.filter(rfcauthor_count__gt=0).distinct() + rfcs_without_rfcauthors = qualifying_rfcs.filter(rfcauthor_count=0).distinct() + + # Second, get the IESG-approved I-Ds excluding any we're already counting as rfcs + qualifying_approval_events = DocEvent.objects.filter( + type='iesg_approved', + time__gte=eligibility_period_start, + time__lte=eligibility_period_end, + ) + qualifying_drafts = Document.objects.filter( + type_id="draft", + docevent__in=qualifying_approval_events, + ).exclude( + relateddocument__relationship_id="became_rfc", + relateddocument__target__in=qualifying_rfcs, + ).distinct() + + return base_qs.filter( + Q(documentauthor__document__in=qualifying_drafts) + | Q(rfcauthor__document__in=rfcs_with_rfcauthors) + | Q(documentauthor__document__in=rfcs_without_rfcauthors) + ).annotate( + document_author_count=Count('documentauthor'), + rfc_author_count=Count("rfcauthor") + ).annotate( + authorship_count=F("document_author_count") + F("rfc_author_count") + ).filter(authorship_count__gte=2) + + def get_threerule_eligibility_querysets(date, base_qs, three_of_five_callable): if not base_qs: base_qs = Person.objects.all() @@ -608,14 +672,7 @@ def get_threerule_eligibility_querysets(date, base_qs, three_of_five_callable): ) ).distinct() - rfc_pks = set(DocEvent.objects.filter(type='published_rfc', time__gte=five_years_ago, time__lte=date_as_dt).values_list('doc__pk', flat=True)) - iesgappr_pks = set(DocEvent.objects.filter(type='iesg_approved', time__gte=five_years_ago, time__lte=date_as_dt).values_list('doc__pk',flat=True)) - qualifying_pks = rfc_pks.union(iesgappr_pks.difference(rfc_pks)) - author_qs = base_qs.filter( - documentauthor__document__pk__in=qualifying_pks - ).annotate( - document_author_count = Count('documentauthor') - ).filter(document_author_count__gte=2) + author_qs = get_qualified_author_queryset(base_qs, five_years_ago, date_as_dt) return three_of_five_qs, officer_qs, author_qs def list_eligible_8989(date, base_qs=None): @@ -691,18 +748,42 @@ def three_of_five_eligible_9389(previous_five, queryset=None): counts[id] += 1 return queryset.filter(pk__in=[id for id, count in counts.items() if count >= 3]) -def suggest_affiliation(person): +def suggest_affiliation(person) -> str: + """Heuristically suggest a current affiliation for a Person""" recent_meeting = person.registration_set.order_by('-meeting__date').first() - affiliation = recent_meeting.affiliation if recent_meeting else '' - if not affiliation: - recent_volunteer = person.volunteer_set.order_by('-nomcom__group__acronym').first() - if recent_volunteer: - affiliation = recent_volunteer.affiliation - if not affiliation: - recent_draft_revision = NewRevisionDocEvent.objects.filter(doc__type_id='draft',doc__documentauthor__person=person).order_by('-time').first() - if recent_draft_revision: - affiliation = recent_draft_revision.doc.documentauthor_set.filter(person=person).first().affiliation - return affiliation + if recent_meeting and recent_meeting.affiliation: + return recent_meeting.affiliation + + recent_volunteer = person.volunteer_set.order_by('-nomcom__group__acronym').first() + if recent_volunteer and recent_volunteer.affiliation: + return recent_volunteer.affiliation + + recent_draft_revision = NewRevisionDocEvent.objects.filter( + doc__type_id="draft", + doc__documentauthor__person=person, + ).order_by("-time").first() + if recent_draft_revision: + draft_author = recent_draft_revision.doc.documentauthor_set.filter( + person=person + ).first() + if draft_author and draft_author.affiliation: + return draft_author.affiliation + + recent_rfc_publication = DocEvent.objects.filter( + Q(doc__documentauthor__person=person) | Q(doc__rfcauthor__person=person), + doc__type_id="rfc", + type="published_rfc", + ).order_by("-time").first() + if recent_rfc_publication: + rfc = recent_rfc_publication.doc + if rfc.rfcauthor_set.exists(): + rfc_author = rfc.rfcauthor_set.filter(person=person).first() + else: + rfc_author = rfc.documentauthor_set.filter(person=person).first() + if rfc_author and rfc_author.affiliation: + return rfc_author.affiliation + return "" + def extract_volunteers(year): nomcom = get_nomcom_by_year(year) diff --git a/ietf/nomcom/views.py b/ietf/nomcom/views.py index c04e13f92b..3f90be5253 100644 --- a/ietf/nomcom/views.py +++ b/ietf/nomcom/views.py @@ -981,7 +981,7 @@ def view_feedback_topic(request, year, topic_id): reviewer = request.user.person last_seen = TopicFeedbackLastSeen.objects.filter(reviewer=reviewer,topic=topic).first() - last_seen_time = (last_seen and last_seen.time) or datetime.datetime(year=1, month=1, day=1, tzinfo=datetime.timezone.utc) + last_seen_time = (last_seen and last_seen.time) or datetime.datetime(year=1, month=1, day=1, tzinfo=datetime.UTC) if last_seen: last_seen.save() else: @@ -1044,7 +1044,7 @@ def view_feedback_nominee(request, year, nominee_id): }) last_seen = FeedbackLastSeen.objects.filter(reviewer=reviewer,nominee=nominee).first() - last_seen_time = (last_seen and last_seen.time) or datetime.datetime(year=1, month=1, day=1, tzinfo=datetime.timezone.utc) + last_seen_time = (last_seen and last_seen.time) or datetime.datetime(year=1, month=1, day=1, tzinfo=datetime.UTC) if last_seen: last_seen.save() else: diff --git a/ietf/person/admin.py b/ietf/person/admin.py index cd8ca2abf1..f46edcf8ae 100644 --- a/ietf/person/admin.py +++ b/ietf/person/admin.py @@ -7,6 +7,7 @@ from ietf.person.models import Email, Alias, Person, PersonalApiKey, PersonEvent, PersonApiKeyEvent, PersonExtResource from ietf.person.name import name_parts +from ietf.utils.admin import SaferStackedInline, SaferTabularInline from ietf.utils.validators import validate_external_resource_value @@ -16,7 +17,7 @@ class EmailAdmin(simple_history.admin.SimpleHistoryAdmin): search_fields = ["address", "person__name", ] admin.site.register(Email, EmailAdmin) -class EmailInline(admin.TabularInline): +class EmailInline(SaferTabularInline): model = Email class AliasAdmin(admin.ModelAdmin): @@ -25,7 +26,7 @@ class AliasAdmin(admin.ModelAdmin): raw_id_fields = ["person"] admin.site.register(Alias, AliasAdmin) -class AliasInline(admin.StackedInline): +class AliasInline(SaferStackedInline): model = Alias class PersonAdmin(simple_history.admin.SimpleHistoryAdmin): diff --git a/ietf/person/forms.py b/ietf/person/forms.py index 81ee362561..7eef8aa17b 100644 --- a/ietf/person/forms.py +++ b/ietf/person/forms.py @@ -1,15 +1,26 @@ -# Copyright The IETF Trust 2018-2020, All Rights Reserved +# Copyright The IETF Trust 2018-2025, All Rights Reserved # -*- coding: utf-8 -*- from django import forms + from ietf.person.models import Person +from ietf.utils.fields import MultiEmailField, NameAddrEmailField class MergeForm(forms.Form): source = forms.IntegerField(label='Source Person ID') target = forms.IntegerField(label='Target Person ID') + def __init__(self, *args, **kwargs): + self.readonly = False + if 'readonly' in kwargs: + self.readonly = kwargs.pop('readonly') + super().__init__(*args, **kwargs) + if self.readonly: + self.fields['source'].widget.attrs['readonly'] = True + self.fields['target'].widget.attrs['readonly'] = True + def clean_source(self): return self.get_person(self.cleaned_data['source']) @@ -21,3 +32,11 @@ def get_person(self, pk): return Person.objects.get(pk=pk) except Person.DoesNotExist: raise forms.ValidationError("ID does not exist") + + +class MergeRequestForm(forms.Form): + to = MultiEmailField() + frm = NameAddrEmailField() + reply_to = MultiEmailField() + subject = forms.CharField() + body = forms.CharField(widget=forms.Textarea) diff --git a/ietf/person/models.py b/ietf/person/models.py index 03cf0c87fb..3ab89289a6 100644 --- a/ietf/person/models.py +++ b/ietf/person/models.py @@ -87,7 +87,7 @@ def short(self): else: prefix, first, middle, last, suffix = self.ascii_parts() return (first and first[0]+"." or "")+(middle or "")+" "+last+(suffix and " "+suffix or "") - def plain_name(self): + def plain_name(self) -> str: if not hasattr(self, '_cached_plain_name'): if self.plain: self._cached_plain_name = self.plain @@ -203,7 +203,10 @@ def has_drafts(self): def rfcs(self): from ietf.doc.models import Document - rfcs = list(Document.objects.filter(documentauthor__person=self, type='rfc')) + # When RfcAuthors are populated, this may over-return if an author is dropped + # from the author list between the final draft and the published RFC. Should + # ignore DocumentAuthors when an RfcAuthor exists for a draft. + rfcs = list(Document.objects.filter(type="rfc").filter(models.Q(documentauthor__person=self)|models.Q(rfcauthor__person=self)).distinct()) rfcs.sort(key=lambda d: d.name ) return rfcs @@ -266,11 +269,16 @@ def available_api_endpoints(self): def cdn_photo_url(self, size=80): if self.photo: if settings.SERVE_CDN_PHOTOS: + if settings.SERVER_MODE != "production": + original_media_dir = settings.MEDIA_URL + settings.MEDIA_URL = "https://www.ietf.org/lib/dt/media/" source_url = self.photo.url if source_url.startswith(settings.IETF_HOST_URL): source_url = source_url[len(settings.IETF_HOST_URL):] elif source_url.startswith('/'): source_url = source_url[1:] + if settings.SERVER_MODE != "production": + settings.MEDIA_URL = original_media_dir return f'{settings.IETF_HOST_URL}cdn-cgi/image/fit=scale-down,width={size},height={size}/{source_url}' else: datatracker_photo_path = urlreverse('ietf.person.views.photo', kwargs={'email_or_name': self.email()}) diff --git a/ietf/person/tests.py b/ietf/person/tests.py index 6326362fd8..f55d8b8a34 100644 --- a/ietf/person/tests.py +++ b/ietf/person/tests.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2014-2022, All Rights Reserved +# Copyright The IETF Trust 2014-2025, All Rights Reserved # -*- coding: utf-8 -*- @@ -10,7 +10,6 @@ from PIL import Image from pyquery import PyQuery - from django.core.exceptions import ValidationError from django.http import HttpRequest from django.test import override_settings @@ -23,6 +22,7 @@ from ietf.community.models import CommunityList from ietf.group.factories import RoleFactory from ietf.group.models import Group +from ietf.message.models import Message from ietf.nomcom.models import NomCom from ietf.nomcom.test_data import nomcom_test_data from ietf.nomcom.factories import NomComFactory, NomineeFactory, NominationFactory, FeedbackFactory, PositionFactory @@ -208,13 +208,13 @@ def test_merge(self): def test_merge_with_params(self): p1 = get_person_no_user() p2 = PersonFactory() - url = urlreverse("ietf.person.views.merge") + "?source={}&target={}".format(p1.pk, p2.pk) + url = urlreverse("ietf.person.views.merge_submit") + "?source={}&target={}".format(p1.pk, p2.pk) login_testing_unauthorized(self, "secretary", url) r = self.client.get(url) self.assertContains(r, 'retaining login', status_code=200) def test_merge_with_params_bad_id(self): - url = urlreverse("ietf.person.views.merge") + "?source=1000&target=2000" + url = urlreverse("ietf.person.views.merge_submit") + "?source=1000&target=2000" login_testing_unauthorized(self, "secretary", url) r = self.client.get(url) self.assertContains(r, 'ID does not exist', status_code=200) @@ -222,7 +222,7 @@ def test_merge_with_params_bad_id(self): def test_merge_post(self): p1 = get_person_no_user() p2 = PersonFactory() - url = urlreverse("ietf.person.views.merge") + url = urlreverse("ietf.person.views.merge_submit") expected_url = urlreverse("ietf.secr.rolodex.views.view", kwargs={'id': p2.pk}) login_testing_unauthorized(self, "secretary", url) data = {'source': p1.pk, 'target': p2.pk} @@ -451,6 +451,30 @@ def test_dots(self): ncchair = RoleFactory(group__acronym='nomcom2020',group__type_id='nomcom',name_id='chair').person self.assertEqual(get_dots(ncchair),['nomcom']) + def test_send_merge_request(self): + empty_outbox() + message_count_before = Message.objects.count() + source = PersonFactory() + target = PersonFactory() + url = urlreverse('ietf.person.views.send_merge_request') + url = url + f'?source={source.pk}&target={target.pk}' + login_testing_unauthorized(self, 'secretary', url) + r = self.client.get(url) + initial = r.context['form'].initial + subject = 'Action requested: Merging possible duplicate IETF Datatracker accounts' + self.assertEqual(initial['to'], ', '.join([source.user.username, target.user.username])) + self.assertEqual(initial['subject'], subject) + self.assertEqual(initial['reply_to'], 'support@ietf.org') + self.assertEqual(r.status_code, 200) + r = self.client.post(url, data=initial) + self.assertEqual(r.status_code, 302) + self.assertEqual(len(outbox), 1) + self.assertIn(source.user.username, outbox[0]['To']) + message_count_after = Message.objects.count() + message = Message.objects.last() + self.assertEqual(message_count_after, message_count_before + 1) + self.assertIn(source.user.username, message.to) + class TaskTests(TestCase): @mock.patch("ietf.person.tasks.log.log") diff --git a/ietf/person/urls.py b/ietf/person/urls.py index 867646fe39..f3eccd04b7 100644 --- a/ietf/person/urls.py +++ b/ietf/person/urls.py @@ -1,8 +1,12 @@ +# Copyright The IETF Trust 2009-2025, All Rights Reserved +# -*- coding: utf-8 -*- from ietf.person import views, ajax from ietf.utils.urls import url urlpatterns = [ url(r'^merge/?$', views.merge), + url(r'^merge/submit/?$', views.merge_submit), + url(r'^merge/send_request/?$', views.send_merge_request), url(r'^search/(?P(person|email))/$', views.ajax_select2_search), url(r'^(?P[0-9]+)/email.json$', ajax.person_email_json), url(r'^(?P[^/]+)$', views.profile), diff --git a/ietf/person/views.py b/ietf/person/views.py index a37b164311..d0b5912431 100644 --- a/ietf/person/views.py +++ b/ietf/person/views.py @@ -1,14 +1,16 @@ -# Copyright The IETF Trust 2012-2020, All Rights Reserved +# Copyright The IETF Trust 2012-2025, All Rights Reserved # -*- coding: utf-8 -*- from io import StringIO, BytesIO from PIL import Image +from django.conf import settings from django.contrib import messages from django.db.models import Q from django.http import HttpResponse, Http404 from django.shortcuts import render, redirect +from django.template.loader import render_to_string from django.utils import timezone import debug # pyflakes:ignore @@ -16,8 +18,9 @@ from ietf.ietfauth.utils import role_required from ietf.person.models import Email, Person from ietf.person.fields import select2_id_name_json -from ietf.person.forms import MergeForm +from ietf.person.forms import MergeForm, MergeRequestForm from ietf.person.utils import handle_users, merge_persons, lookup_persons +from ietf.utils.mail import send_mail_text def ajax_select2_search(request, model_name): @@ -98,16 +101,19 @@ def photo(request, email_or_name): @role_required("Secretariat") def merge(request): form = MergeForm() - method = 'get' + return render(request, 'person/merge.html', {'form': form}) + + +@role_required("Secretariat") +def merge_submit(request): change_details = '' warn_messages = [] source = None target = None if request.method == "GET": - form = MergeForm() if request.GET: - form = MergeForm(request.GET) + form = MergeForm(request.GET, readonly=True) if form.is_valid(): source = form.cleaned_data.get('source') target = form.cleaned_data.get('target') @@ -116,12 +122,9 @@ def merge(request): if source.user.last_login and target.user.last_login and source.user.last_login > target.user.last_login: warn_messages.append('WARNING: The most recently used login is being deleted!') change_details = handle_users(source, target, check_only=True) - method = 'post' - else: - method = 'get' if request.method == "POST": - form = MergeForm(request.POST) + form = MergeForm(request.POST, readonly=True) if form.is_valid(): source = form.cleaned_data.get('source') source_id = source.id @@ -136,11 +139,72 @@ def merge(request): messages.error(request, output) return redirect('ietf.secr.rolodex.views.view', id=target.pk) - return render(request, 'person/merge.html', { + return render(request, 'person/merge_submit.html', { 'form': form, - 'method': method, 'change_details': change_details, 'source': source, 'target': target, 'warn_messages': warn_messages, }) + + +@role_required("Secretariat") +def send_merge_request(request): + if request.method == 'GET': + merge_form = MergeForm(request.GET) + if merge_form.is_valid(): + source = merge_form.cleaned_data['source'] + target = merge_form.cleaned_data['target'] + to = [] + if source.email(): + to.append(source.email().address) + if target.email(): + to.append(target.email().address) + if source.user: + source_account = source.user.username + else: + source_account = source.email() + if target.user: + target_account = target.user.username + else: + target_account = target.email() + sender_name = request.user.person.name + subject = 'Action requested: Merging possible duplicate IETF Datatracker accounts' + context = { + 'source_account': source_account, + 'target_account': target_account, + 'sender_name': sender_name, + } + body = render_to_string('person/merge_request_email.txt', context) + initial = { + 'to': ', '.join(to), + 'frm': settings.DEFAULT_FROM_EMAIL, + 'reply_to': 'support@ietf.org', + 'subject': subject, + 'body': body, + 'by': request.user.person.pk, + } + form = MergeRequestForm(initial=initial) + else: + messages.error(request, "Error requesting merge email: " + merge_form.errors.as_text()) + return redirect("ietf.person.views.merge") + + if request.method == 'POST': + form = MergeRequestForm(request.POST) + if form.is_valid(): + extra = {"Reply-To": form.cleaned_data.get("reply_to")} + send_mail_text( + request, + form.cleaned_data.get("to"), + form.cleaned_data.get("frm"), + form.cleaned_data.get("subject"), + form.cleaned_data.get("body"), + extra=extra, + ) + + messages.success(request, "The merge confirmation email was sent.") + return redirect("ietf.person.views.merge") + + return render(request, "person/send_merge_request.html", { + "form": form, + }) diff --git a/ietf/secr/meetings/views.py b/ietf/secr/meetings/views.py index 47f7b7ffa5..1f6f2f3297 100644 --- a/ietf/secr/meetings/views.py +++ b/ietf/secr/meetings/views.py @@ -1,4 +1,4 @@ -# Copyright The IETF Trust 2007-2023, All Rights Reserved +# Copyright The IETF Trust 2007-2025, All Rights Reserved # -*- coding: utf-8 -*- import datetime @@ -20,12 +20,12 @@ from ietf.meeting.helpers import make_materials_directories, populate_important_dates from ietf.meeting.models import Meeting, Session, Room, TimeSlot, SchedTimeSessAssignment, Schedule, SchedulingEvent from ietf.meeting.utils import add_event_info_to_session_qs +from ietf.meeting.views_session_request import get_initial_session from ietf.name.models import SessionStatusName from ietf.group.models import Group, GroupEvent from ietf.secr.meetings.forms import ( BaseMeetingRoomFormSet, MeetingModelForm, MeetingSelectForm, MeetingRoomForm, MiscSessionForm, TimeSlotForm, RegularSessionEditForm, MeetingRoomOptionsForm ) -from ietf.secr.sreq.views import get_initial_session from ietf.secr.utils.meeting import get_session, get_timeslot from ietf.mailtrigger.utils import gather_address_lists from ietf.utils.timezone import make_aware diff --git a/ietf/secr/sreq/forms.py b/ietf/secr/sreq/forms.py deleted file mode 100644 index 4a0f449b2a..0000000000 --- a/ietf/secr/sreq/forms.py +++ /dev/null @@ -1,333 +0,0 @@ -# Copyright The IETF Trust 2013-2022, All Rights Reserved -# -*- coding: utf-8 -*- - - -from django import forms -from django.template.defaultfilters import pluralize - -import debug # pyflakes:ignore - -from ietf.name.models import TimerangeName, ConstraintName -from ietf.group.models import Group -from ietf.meeting.forms import sessiondetailsformset_factory -from ietf.meeting.models import ResourceAssociation, Constraint -from ietf.person.fields import SearchablePersonsField -from ietf.person.models import Person -from ietf.utils.fields import ModelMultipleChoiceField -from ietf.utils.html import clean_text_field -from ietf.utils import log - -# ------------------------------------------------- -# Globals -# ------------------------------------------------- - -NUM_SESSION_CHOICES = (('','--Please select'),('1','1'),('2','2')) -SESSION_TIME_RELATION_CHOICES = (('', 'No preference'),) + Constraint.TIME_RELATION_CHOICES -JOINT_FOR_SESSION_CHOICES = (('1', 'First session'), ('2', 'Second session'), ('3', 'Third session'), ) - -# ------------------------------------------------- -# Helper Functions -# ------------------------------------------------- -def allowed_conflicting_groups(): - return Group.objects.filter(type__in=['wg', 'ag', 'rg', 'rag', 'program', 'edwg'], state__in=['bof', 'proposed', 'active']) - -def check_conflict(groups, source_group): - ''' - Takes a string which is a list of group acronyms. Checks that they are all active groups - ''' - # convert to python list (allow space or comma separated lists) - items = groups.replace(',',' ').split() - active_groups = allowed_conflicting_groups() - for group in items: - if group == source_group.acronym: - raise forms.ValidationError("Cannot declare a conflict with the same group: %s" % group) - - if not active_groups.filter(acronym=group): - raise forms.ValidationError("Invalid or inactive group acronym: %s" % group) - -# ------------------------------------------------- -# Forms -# ------------------------------------------------- - -class GroupSelectForm(forms.Form): - group = forms.ChoiceField() - - def __init__(self,*args,**kwargs): - choices = kwargs.pop('choices') - super(GroupSelectForm, self).__init__(*args,**kwargs) - self.fields['group'].widget.choices = choices - - -class NameModelMultipleChoiceField(ModelMultipleChoiceField): - def label_from_instance(self, name): - return name.desc - - -class SessionForm(forms.Form): - num_session = forms.ChoiceField(choices=NUM_SESSION_CHOICES) - # session fields are added in __init__() - session_time_relation = forms.ChoiceField(choices=SESSION_TIME_RELATION_CHOICES, required=False) - attendees = forms.IntegerField() - # FIXME: it would cleaner to have these be - # ModelMultipleChoiceField, and just customize the widgetry, that - # way validation comes for free (applies to this CharField and the - # constraints dynamically instantiated in __init__()) - joint_with_groups = forms.CharField(max_length=255,required=False) - joint_with_groups_selector = forms.ChoiceField(choices=[], required=False) # group select widget for prev field - joint_for_session = forms.ChoiceField(choices=JOINT_FOR_SESSION_CHOICES, required=False) - comments = forms.CharField(max_length=200,required=False) - third_session = forms.BooleanField(required=False) - resources = forms.MultipleChoiceField(widget=forms.CheckboxSelectMultiple,required=False) - bethere = SearchablePersonsField(label="Must be present", required=False) - timeranges = NameModelMultipleChoiceField(widget=forms.CheckboxSelectMultiple, required=False, - queryset=TimerangeName.objects.all()) - adjacent_with_wg = forms.ChoiceField(required=False) - send_notifications = forms.BooleanField(label="Send notification emails?", required=False, initial=False) - - def __init__(self, group, meeting, data=None, *args, **kwargs): - self.hidden = kwargs.pop('hidden', False) - self.notifications_optional = kwargs.pop('notifications_optional', False) - - self.group = group - formset_class = sessiondetailsformset_factory(max_num=3 if group.features.acts_like_wg else 50) - self.session_forms = formset_class(group=self.group, meeting=meeting, data=data) - super(SessionForm, self).__init__(data=data, *args, **kwargs) - if not self.notifications_optional: - self.fields['send_notifications'].widget = forms.HiddenInput() - - # Allow additional sessions for non-wg-like groups - if not self.group.features.acts_like_wg: - self.fields['num_session'].choices = ((n, str(n)) for n in range(1, 51)) - - self.fields['comments'].widget = forms.Textarea(attrs={'rows':'3','cols':'65'}) - - other_groups = list(allowed_conflicting_groups().exclude(pk=group.pk).values_list('acronym', 'acronym').order_by('acronym')) - self.fields['adjacent_with_wg'].choices = [('', '--No preference')] + other_groups - group_acronym_choices = [('','--Select WG(s)')] + other_groups - self.fields['joint_with_groups_selector'].choices = group_acronym_choices - - # Set up constraints for the meeting - self._wg_field_data = [] - for constraintname in meeting.group_conflict_types.all(): - # two fields for each constraint: a CharField for the group list and a selector to add entries - constraint_field = forms.CharField(max_length=255, required=False) - constraint_field.widget.attrs['data-slug'] = constraintname.slug - constraint_field.widget.attrs['data-constraint-name'] = str(constraintname).title() - self._add_widget_class(constraint_field.widget, 'wg_constraint') - - selector_field = forms.ChoiceField(choices=group_acronym_choices, required=False) - selector_field.widget.attrs['data-slug'] = constraintname.slug # used by onchange handler - self._add_widget_class(selector_field.widget, 'wg_constraint_selector') - - cfield_id = 'constraint_{}'.format(constraintname.slug) - cselector_id = 'wg_selector_{}'.format(constraintname.slug) - # keep an eye out for field name conflicts - log.assertion('cfield_id not in self.fields') - log.assertion('cselector_id not in self.fields') - self.fields[cfield_id] = constraint_field - self.fields[cselector_id] = selector_field - self._wg_field_data.append((constraintname, cfield_id, cselector_id)) - - # Show constraints that are not actually used by the meeting so these don't get lost - self._inactive_wg_field_data = [] - inactive_cnames = ConstraintName.objects.filter( - is_group_conflict=True # Only collect group conflicts... - ).exclude( - meeting=meeting # ...that are not enabled for this meeting... - ).filter( - constraint__source=group, # ...but exist for this group... - constraint__meeting=meeting, # ... at this meeting. - ).distinct() - - for inactive_constraint_name in inactive_cnames: - field_id = 'delete_{}'.format(inactive_constraint_name.slug) - self.fields[field_id] = forms.BooleanField(required=False, label='Delete this conflict', help_text='Delete this inactive conflict?') - constraints = group.constraint_source_set.filter(meeting=meeting, name=inactive_constraint_name) - self._inactive_wg_field_data.append( - (inactive_constraint_name, - ' '.join([c.target.acronym for c in constraints]), - field_id) - ) - - self.fields['joint_with_groups_selector'].widget.attrs['onchange'] = "document.form_post.joint_with_groups.value=document.form_post.joint_with_groups.value + ' ' + this.options[this.selectedIndex].value; return 1;" - self.fields["resources"].choices = [(x.pk,x.desc) for x in ResourceAssociation.objects.filter(name__used=True).order_by('name__order') ] - - if self.hidden: - # replace all the widgets to start... - for key in list(self.fields.keys()): - self.fields[key].widget = forms.HiddenInput() - # re-replace a couple special cases - self.fields['resources'].widget = forms.MultipleHiddenInput() - self.fields['timeranges'].widget = forms.MultipleHiddenInput() - # and entirely replace bethere - no need to support searching if input is hidden - self.fields['bethere'] = ModelMultipleChoiceField( - widget=forms.MultipleHiddenInput, required=False, - queryset=Person.objects.all(), - ) - - def wg_constraint_fields(self): - """Iterates over wg constraint fields - - Intended for use in the template. - """ - for cname, cfield_id, cselector_id in self._wg_field_data: - yield cname, self[cfield_id], self[cselector_id] - - def wg_constraint_count(self): - """How many wg constraints are there?""" - return len(self._wg_field_data) - - def wg_constraint_field_ids(self): - """Iterates over wg constraint field IDs""" - for cname, cfield_id, _ in self._wg_field_data: - yield cname, cfield_id - - def inactive_wg_constraints(self): - for cname, value, field_id in self._inactive_wg_field_data: - yield cname, value, self[field_id] - - def inactive_wg_constraint_count(self): - return len(self._inactive_wg_field_data) - - def inactive_wg_constraint_field_ids(self): - """Iterates over wg constraint field IDs""" - for cname, _, field_id in self._inactive_wg_field_data: - yield cname, field_id - - @staticmethod - def _add_widget_class(widget, new_class): - """Add a new class, taking care in case some already exist""" - existing_classes = widget.attrs.get('class', '').split() - widget.attrs['class'] = ' '.join(existing_classes + [new_class]) - - def _join_conflicts(self, cleaned_data, slugs): - """Concatenate constraint fields from cleaned data into a single list""" - conflicts = [] - for cname, cfield_id, _ in self._wg_field_data: - if cname.slug in slugs and cfield_id in cleaned_data: - groups = cleaned_data[cfield_id] - # convert to python list (allow space or comma separated lists) - items = groups.replace(',',' ').split() - conflicts.extend(items) - return conflicts - - def _validate_duplicate_conflicts(self, cleaned_data): - """Validate that no WGs appear in more than one constraint that does not allow duplicates - - Raises ValidationError - """ - # Only the older constraints (conflict, conflic2, conflic3) need to be mutually exclusive. - all_conflicts = self._join_conflicts(cleaned_data, ['conflict', 'conflic2', 'conflic3']) - seen = [] - duplicated = [] - errors = [] - for c in all_conflicts: - if c not in seen: - seen.append(c) - elif c not in duplicated: # only report once - duplicated.append(c) - errors.append(forms.ValidationError('%s appears in conflicts more than once' % c)) - return errors - - def clean_joint_with_groups(self): - groups = self.cleaned_data['joint_with_groups'] - check_conflict(groups, self.group) - return groups - - def clean_comments(self): - return clean_text_field(self.cleaned_data['comments']) - - def clean_bethere(self): - bethere = self.cleaned_data["bethere"] - if bethere: - extra = set( - Person.objects.filter( - role__group=self.group, role__name__in=["chair", "ad"] - ) - & bethere - ) - if extra: - extras = ", ".join(e.name for e in extra) - raise forms.ValidationError( - ( - f"Please remove the following person{pluralize(len(extra))}, the system " - f"tracks their availability due to their role{pluralize(len(extra))}: {extras}." - ) - ) - return bethere - - def clean_send_notifications(self): - return True if not self.notifications_optional else self.cleaned_data['send_notifications'] - - def is_valid(self): - return super().is_valid() and self.session_forms.is_valid() - - def clean(self): - super(SessionForm, self).clean() - self.session_forms.clean() - - data = self.cleaned_data - - # Validate the individual conflict fields - for _, cfield_id, _ in self._wg_field_data: - try: - check_conflict(data[cfield_id], self.group) - except forms.ValidationError as e: - self.add_error(cfield_id, e) - - # Skip remaining tests if individual field tests had errors, - if self.errors: - return data - - # error if conflicts contain disallowed dupes - for error in self._validate_duplicate_conflicts(data): - self.add_error(None, error) - - # Verify expected number of session entries are present - num_sessions_with_data = len(self.session_forms.forms_to_keep) - num_sessions_expected = -1 - try: - num_sessions_expected = int(data.get('num_session', '')) - except ValueError: - self.add_error('num_session', 'Invalid value for number of sessions') - if num_sessions_with_data < num_sessions_expected: - self.add_error('num_session', 'Must provide data for all sessions') - - # if default (empty) option is selected, cleaned_data won't include num_session key - if num_sessions_expected != 2 and num_sessions_expected is not None: - if data.get('session_time_relation'): - self.add_error( - 'session_time_relation', - forms.ValidationError('Time between sessions can only be used when two sessions are requested.') - ) - - joint_session = data.get('joint_for_session', '') - if joint_session != '': - joint_session = int(joint_session) - if joint_session > num_sessions_with_data: - self.add_error( - 'joint_for_session', - forms.ValidationError( - f'Session {joint_session} can not be the joint session, the session has not been requested.' - ) - ) - - return data - - @property - def media(self): - # get media for our formset - return super().media + self.session_forms.media + forms.Media(js=('secr/js/session_form.js',)) - - -# Used for totally virtual meetings during COVID-19 to omit the expected -# number of attendees since there were no room size limitations -# -# class VirtualSessionForm(SessionForm): -# '''A SessionForm customized for special virtual meeting requirements''' -# attendees = forms.IntegerField(required=False) - - -class ToolStatusForm(forms.Form): - message = forms.CharField(widget=forms.Textarea(attrs={'rows':'3','cols':'80'}), strip=False) - diff --git a/ietf/secr/sreq/templatetags/__init__.py b/ietf/secr/sreq/templatetags/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/ietf/secr/sreq/urls.py b/ietf/secr/sreq/urls.py deleted file mode 100644 index 7e0db8117a..0000000000 --- a/ietf/secr/sreq/urls.py +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright The IETF Trust 2007-2019, All Rights Reserved - -from django.conf import settings - -from ietf.secr.sreq import views -from ietf.utils.urls import url - -urlpatterns = [ - url(r'^$', views.main), - url(r'^status/$', views.tool_status), - url(r'^%(acronym)s/$' % settings.URL_REGEXPS, views.view), - url(r'^(?P[A-Za-z0-9_\-\+]+)/%(acronym)s/view/$' % settings.URL_REGEXPS, views.view), - url(r'^%(acronym)s/approve/$' % settings.URL_REGEXPS, views.approve), - url(r'^%(acronym)s/cancel/$' % settings.URL_REGEXPS, views.cancel), - url(r'^%(acronym)s/confirm/$' % settings.URL_REGEXPS, views.confirm), - url(r'^%(acronym)s/edit/$' % settings.URL_REGEXPS, views.edit), - url(r'^%(acronym)s/new/$' % settings.URL_REGEXPS, views.new), - url(r'^%(acronym)s/no_session/$' % settings.URL_REGEXPS, views.no_session), - url(r'^(?P[A-Za-z0-9_\-\+]+)/%(acronym)s/edit/$' % settings.URL_REGEXPS, views.edit), -] diff --git a/ietf/secr/telechat/tests.py b/ietf/secr/telechat/tests.py index 39949b83a2..91ccde2187 100644 --- a/ietf/secr/telechat/tests.py +++ b/ietf/secr/telechat/tests.py @@ -13,6 +13,7 @@ IndividualDraftFactory, ConflictReviewFactory) from ietf.doc.models import BallotDocEvent, BallotType, BallotPositionDocEvent, State, Document from ietf.doc.utils import update_telechat, create_ballot_if_not_open +from ietf.meeting.factories import MeetingFactory from ietf.utils.test_utils import TestCase from ietf.utils.timezone import date_today, datetime_today from ietf.iesg.models import TelechatDate @@ -25,6 +26,26 @@ def augment_data(): TelechatDate.objects.create(date=date_today()) +class SecrUrlTests(TestCase): + def test_urls(self): + MeetingFactory(type_id='ietf', date=date_today()) + + # check public options + response = self.client.get("/secr/") + self.assertEqual(response.status_code, 200) + q = PyQuery(response.content) + links = q('div.secr-menu a') + self.assertEqual(len(links), 1) + self.assertEqual(PyQuery(links[0]).text(), 'Announcements') + + # check secretariat only options + self.client.login(username="secretary", password="secretary+password") + response = self.client.get("/secr/") + self.assertEqual(response.status_code, 200) + q = PyQuery(response.content) + links = q('div.secr-menu a') + self.assertEqual(len(links), 4) + class SecrTelechatTestCase(TestCase): def test_main(self): "Main Test" @@ -235,7 +256,7 @@ def test_doc_detail_post_update_state_action_holder_automation(self): self.assertEqual(response.status_code,302) draft = Document.objects.get(name=draft.name) self.assertEqual(draft.get_state('draft-iesg').slug,'defer') - self.assertCountEqual(draft.action_holders.all(), [draft.ad] + draft.authors()) + self.assertCountEqual(draft.action_holders.all(), [draft.ad] + draft.author_persons()) self.assertEqual(draft.docevent_set.filter(type='changed_action_holders').count(), 1) # Removing need-rev should remove authors @@ -252,7 +273,7 @@ def test_doc_detail_post_update_state_action_holder_automation(self): # Setting to approved should remove all action holders # noinspection DjangoOrm - draft.action_holders.add(*(draft.authors())) # add() with through model ok in Django 2.2+ + draft.action_holders.add(*(draft.author_persons())) # add() with through model ok in Django 2.2+ response = self.client.post(url,{ 'submit': 'update_state', 'state': State.objects.get(type_id='draft-iesg', slug='approved').pk, diff --git a/ietf/secr/templates/includes/activities.html b/ietf/secr/templates/includes/activities.html deleted file mode 100644 index 1304b7c48d..0000000000 --- a/ietf/secr/templates/includes/activities.html +++ /dev/null @@ -1,23 +0,0 @@ -

Activities Log

-
- - - - - - - - - - - {% for entry in activities %} - - - - - - - {% endfor %} - -
DateTimeActionName
{{ entry.act_date }}{{ entry.act_time }}{{ entry.activity }}{{ entry.act_by }}
-
diff --git a/ietf/secr/templates/includes/buttons_next_cancel.html b/ietf/secr/templates/includes/buttons_next_cancel.html deleted file mode 100644 index 95d25f55bc..0000000000 --- a/ietf/secr/templates/includes/buttons_next_cancel.html +++ /dev/null @@ -1,6 +0,0 @@ -
-
    -
    • -
    • -
-
diff --git a/ietf/secr/templates/includes/buttons_submit_cancel.html b/ietf/secr/templates/includes/buttons_submit_cancel.html deleted file mode 100644 index df40c98255..0000000000 --- a/ietf/secr/templates/includes/buttons_submit_cancel.html +++ /dev/null @@ -1,6 +0,0 @@ -
-
    -
    • -
    • -
-
diff --git a/ietf/secr/templates/includes/sessions_footer.html b/ietf/secr/templates/includes/sessions_footer.html deleted file mode 100755 index 2a26440047..0000000000 --- a/ietf/secr/templates/includes/sessions_footer.html +++ /dev/null @@ -1,5 +0,0 @@ - \ No newline at end of file diff --git a/ietf/secr/templates/includes/sessions_request_form.html b/ietf/secr/templates/includes/sessions_request_form.html deleted file mode 100755 index 61b1673357..0000000000 --- a/ietf/secr/templates/includes/sessions_request_form.html +++ /dev/null @@ -1,130 +0,0 @@ -* Required Field -
{% csrf_token %} - {{ form.session_forms.management_form }} - {% if form.non_field_errors %} - {{ form.non_field_errors }} - {% endif %} - - - - - - {% if group.features.acts_like_wg %} - - {% if not is_virtual %} - - {% endif %} - - {% else %}{# else not group.features.acts_like_wg #} - {% for session_form in form.session_forms %} - - {% endfor %} - {% endif %} - - - - - - - - - - {% if not is_virtual %} - - - - - - - - - - - - - - - - - - - {% endif %} - - - - - - {% if form.notifications_optional %} - - - - - {% endif %} - -
Working Group Name:{{ group.name }} ({{ group.acronym }})
Area Name:{% if group.parent %}{{ group.parent.name }} ({{ group.parent.acronym }}){% endif %}
Number of Sessions:*{{ form.num_session.errors }}{{ form.num_session }}
Session 1:*{% include 'meeting/session_details_form.html' with form=form.session_forms.0 hide_onsite_tool_prompt=True only %}
Session 2:*{% include 'meeting/session_details_form.html' with form=form.session_forms.1 hide_onsite_tool_prompt=True only %}
Time between two sessions:{{ form.session_time_relation.errors }}{{ form.session_time_relation }}
Additional Session Request:{{ form.third_session }} Check this box to request an additional session.
- Additional slot may be available after agenda scheduling has closed and with the approval of an Area Director.
-
- Third Session: - {% include 'meeting/session_details_form.html' with form=form.session_forms.2 hide_onsite_tool_prompt=True only %} -
-
Session {{ forloop.counter }}:*{% include 'meeting/session_details_form.html' with form=session_form only %}
Number of Attendees:{% if not is_virtual %}*{% endif %}{{ form.attendees.errors }}{{ form.attendees }}
Participants who must be present: - {{ form.bethere.errors }} - {{ form.bethere }} -

- Do not include Area Directors and WG Chairs; the system already tracks their availability. -

-
Conflicts to Avoid: - - - - - - - {% for cname, cfield, cselector in form.wg_constraint_fields %} - - {% if forloop.first %}{% endif %} - - - - {% empty %}{# shown if there are no constraint fields #} - - {% endfor %} - {% if form.inactive_wg_constraints %} - {% for cname, value, field in form.inactive_wg_constraints %} - - {% if forloop.first %} - - {% endif %} - - - - {% endfor %} - {% endif %} - - - - - -
Other WGs that included {{ group.name }} in their conflict lists:{{ session_conflicts.inbound|default:"None" }}
WG Sessions:
You may select multiple WGs within each category		{{ cname|title }}{{ cselector }} -
- {{ cfield.errors }}{{ cfield }} -
No constraints are enabled for this meeting.
- Disabled for this meeting - {{ cname|title }}
{{ field }} {{ field.label }}
BOF Sessions:If the sessions can not be found in the fields above, please enter free form requests in the Special Requests field below.
-
Resources requested: - {{ form.resources.errors }} {{ form.resources }} -
Times during which this WG can not meet:
Please explain any selections in Special Requests below.		{{ form.timeranges.errors }}{{ form.timeranges }}
- Plan session adjacent with another WG:
- (Immediately before or after another WG, no break in between, in the same room.) - 		{{ form.adjacent_with_wg.errors }}{{ form.adjacent_with_wg }}
- Joint session with:
- (To request one session for multiple WGs together.) - 		To request a joint session with another group, please contact the secretariat.
Special Requests:
 
i.e. restrictions on meeting times / days, etc.
(limit 200 characters)		{{ form.comments.errors }}{{ form.comments }}
{{ form.send_notifications.label }}{{ form.send_notifications.errors }}{{ form.send_notifications }}
- -
-
    -
    • -
    • -
-
-
\ No newline at end of file diff --git a/ietf/secr/templates/includes/sessions_request_view.html b/ietf/secr/templates/includes/sessions_request_view.html deleted file mode 100644 index bc6aef0611..0000000000 --- a/ietf/secr/templates/includes/sessions_request_view.html +++ /dev/null @@ -1,73 +0,0 @@ -{% load ams_filters %} - - - - - - {% if form %} - {% include 'includes/sessions_request_view_formset.html' with formset=form.session_forms group=group session=session only %} - {% else %} - {% include 'includes/sessions_request_view_session_set.html' with session_set=sessions group=group session=session only %} - {% endif %} - - - - - - - - - - {% if not is_virtual %} - - - - - {% endif %} - - - - - - - - - {% if not is_virtual %} - - - - - - - - - {% endif %} - - {% if form and form.notifications_optional %} - - - - - {% endif %} - -
Working Group Name:{{ group.name }} ({{ group.acronym }})
Area Name:{{ group.parent }}
Number of Sessions Requested:{% if session.third_session %}3{% else %}{{ session.num_session }}{% endif %}
Number of Attendees:{{ session.attendees }}
Conflicts to Avoid: - {% if session_conflicts.outbound %} - - - {% for conflict in session_conflicts.outbound %} - - {% endfor %} - -
{{ conflict.name|title }}: {{ conflict.groups }}
- {% else %}None{% endif %} -
Other WGs that included {{ group }} in their conflict list:{% if session_conflicts.inbound %}{{ session_conflicts.inbound }}{% else %}None so far{% endif %}
Resources requested:{% if session.resources %}
    {% for resource in session.resources %}
  • {{ resource.desc }}
    • {% endfor %}
{% else %}None so far{% endif %}
Participants who must be present:{% if session.bethere %}
    {% for person in session.bethere %}
  • {{ person }}
    • {% endfor %}
{% else %}None{% endif %}
Can not meet on:{% if session.timeranges_display %}{{ session.timeranges_display|join:', ' }}{% else %}No constraints{% endif %}
Adjacent with WG:{{ session.adjacent_with_wg|default:'No preference' }}
Joint session: - {% if session.joint_with_groups %} - {{ session.joint_for_session_display }} with: {{ session.joint_with_groups }} - {% else %} - Not a joint session - {% endif %} -
Special Requests:{{ session.comments }}
- {{ form.send_notifications.label}} - - {% if form.cleaned_data.send_notifications %}Yes{% else %}No{% endif %} -
\ No newline at end of file diff --git a/ietf/secr/templates/includes/sessions_request_view_formset.html b/ietf/secr/templates/includes/sessions_request_view_formset.html deleted file mode 100644 index 80cad8d829..0000000000 --- a/ietf/secr/templates/includes/sessions_request_view_formset.html +++ /dev/null @@ -1,32 +0,0 @@ -{% load ams_filters %}{# keep this in sync with sessions_request_view_session_set.html #} -{% for sess_form in formset %}{% if sess_form.cleaned_data and not sess_form.cleaned_data.DELETE %} - - Session {{ forloop.counter }}: - -
-
Length
-
{{ sess_form.cleaned_data.requested_duration.total_seconds|display_duration }}
- {% if sess_form.cleaned_data.name %} -
Name
-
{{ sess_form.cleaned_data.name }}
{% endif %} - {% if sess_form.cleaned_data.purpose.slug != 'regular' %} -
Purpose
-
- {{ sess_form.cleaned_data.purpose }} - {% if sess_form.cleaned_data.purpose.timeslot_types|length > 1 %}({{ sess_form.cleaned_data.type }} - ){% endif %} -
-
Onsite tool?
-
{{ sess_form.cleaned_data.has_onsite_tool|yesno }}
- {% endif %} -
- - - {% if group.features.acts_like_wg and forloop.counter == 2 and not is_virtual %} - - Time between sessions: - {% if session.session_time_relation_display %}{{ session.session_time_relation_display }}{% else %}No - preference{% endif %} - - {% endif %} -{% endif %}{% endfor %} \ No newline at end of file diff --git a/ietf/secr/templates/includes/sessions_request_view_session_set.html b/ietf/secr/templates/includes/sessions_request_view_session_set.html deleted file mode 100644 index a434b9d22b..0000000000 --- a/ietf/secr/templates/includes/sessions_request_view_session_set.html +++ /dev/null @@ -1,32 +0,0 @@ -{% load ams_filters %}{# keep this in sync with sessions_request_view_formset.html #} -{% for sess in session_set %} - - Session {{ forloop.counter }}: - -
-
Length
-
{{ sess.requested_duration.total_seconds|display_duration }}
- {% if sess.name %} -
Name
-
{{ sess.name }}
{% endif %} - {% if sess.purpose.slug != 'regular' %} -
Purpose
-
- {{ sess.purpose }} - {% if sess.purpose.timeslot_types|length > 1 %}({{ sess.type }} - ){% endif %} -
-
Onsite tool?
-
{{ sess.has_onsite_tool|yesno }}
- {% endif %} -
- - - {% if group.features.acts_like_wg and forloop.counter == 2 and not is_virtual %} - - Time between sessions: - {% if session.session_time_relation_display %}{{ session.session_time_relation_display }}{% else %}No - preference{% endif %} - - {% endif %} -{% endfor %} \ No newline at end of file diff --git a/ietf/secr/templates/index.html b/ietf/secr/templates/index.html index 05fa3db41f..9ea7021279 100644 --- a/ietf/secr/templates/index.html +++ b/ietf/secr/templates/index.html @@ -1,11 +1,11 @@ -{# Copyright The IETF Trust 2007, All Rights Reserved #} +{# Copyright The IETF Trust 2007-2025, All Rights Reserved #} {% extends "base.html" %} {% load static %} {% load ietf_filters %} {% block title %}Secretariat Dashboard{% endblock %} {% block content %}

Secretariat Dashboard

-
+
{% if user|has_role:"Secretariat" %}

IESG

    @@ -20,12 +20,10 @@

    IDs and WGs Process

    Meetings and Proceedings

    {% else %} {% endif %} diff --git a/ietf/secr/templates/sreq/confirm.html b/ietf/secr/templates/sreq/confirm.html deleted file mode 100755 index 025375af32..0000000000 --- a/ietf/secr/templates/sreq/confirm.html +++ /dev/null @@ -1,57 +0,0 @@ -{% extends "base_site.html" %} -{% load static %} - -{% block title %}Sessions - Confirm{% endblock %} - -{% block extrastyle %} - -{% endblock %} - -{% block extrahead %}{{ block.super }} - - {{ form.media }} -{% endblock %} - -{% block breadcrumbs %}{{ block.super }} - » Sessions - » New - » Session Request Confirmation -{% endblock %} - -{% block content %} - -
    -

    Sessions - Confirm

    - - {% include "includes/sessions_request_view.html" %} - - {% if group.features.acts_like_wg and form.session_forms.forms_to_keep|length > 2 %} -
    -

    - - Note: Your request for a third session must be approved by an area director before - being submitted to agenda@ietf.org. Click "Submit" below to email an approval - request to the area directors. - -

    -
    - {% endif %} - -
    - {% csrf_token %} - {{ form }} - {{ form.session_forms.management_form }} - {% for sf in form.session_forms %} - {% include 'meeting/session_details_form.html' with form=sf hidden=True only %} - {% endfor %} - {% include "includes/buttons_submit_cancel.html" %} -
    - -
    - -{% endblock %} \ No newline at end of file diff --git a/ietf/secr/templates/sreq/edit.html b/ietf/secr/templates/sreq/edit.html deleted file mode 100755 index f6e62104b0..0000000000 --- a/ietf/secr/templates/sreq/edit.html +++ /dev/null @@ -1,39 +0,0 @@ -{% extends "base_site.html" %} -{% load static %} -{% block title %}Sessions - Edit{% endblock %} - -{% block extrahead %}{{ block.super }} - - - {{ form.media }} - -{% endblock %} - -{% block breadcrumbs %}{{ block.super }} - » Sessions - » {{ group.acronym }} - » Edit -{% endblock %} - -{% block instructions %} - Instructions -{% endblock %} - -{% block content %} -
    -

    IETF {{ meeting.number }}: Edit Session Request

    -
    -
    - - {% include "includes/sessions_request_form.html" %} - -
    -
    -{% endblock %} - -{% block footer-extras %} - {% include "includes/sessions_footer.html" %} -{% endblock %} \ No newline at end of file diff --git a/ietf/secr/templates/sreq/locked.html b/ietf/secr/templates/sreq/locked.html deleted file mode 100755 index c27cf578ed..0000000000 --- a/ietf/secr/templates/sreq/locked.html +++ /dev/null @@ -1,30 +0,0 @@ -{% extends "base_site.html" %} -{% load static %} - -{% block title %}Sessions{% endblock %} - -{% block extrahead %}{{ block.super }} - -{% endblock %} - -{% block breadcrumbs %}{{ block.super }} - » Sessions (Locked) -{% endblock %} - -{% block content %} -

    » View list of timeslot requests

    -
    -

    Sessions - Status

    - -

    {{ message }}

    - -
    -
      -
      • -
    -
    - - -
    - -{% endblock %} \ No newline at end of file diff --git a/ietf/secr/templates/sreq/main.html b/ietf/secr/templates/sreq/main.html deleted file mode 100755 index a6695cd4f3..0000000000 --- a/ietf/secr/templates/sreq/main.html +++ /dev/null @@ -1,65 +0,0 @@ -{% extends "base_site.html" %} -{% load ietf_filters %} -{% load static %} - -{% block title %}Sessions{% endblock %} - -{% block extrahead %}{{ block.super }} - -{% endblock %} - -{% block breadcrumbs %}{{ block.super }} - » Sessions -{% endblock %} -{% block instructions %} - Instructions -{% endblock %} - -{% block content %} -

    » View list of timeslot requests

    -
    -

    - Sessions Request Tool: IETF {{ meeting.number }} - {% if user|has_role:"Secretariat" %} - {% if is_locked %} - Tool Status: Locked - {% else %} - Tool Status: Unlocked - {% endif %} - {% endif %} -

    -
    -

    Request New Session

    -

    The list below includes those working groups that you currently chair which do not already have a session scheduled. You can click on an acronym to initiate a request for a new session at the upcoming IETF meeting. Click "Group will not meet" to send a notification that the group does not plan to meet.

    -
      - {% for group in unscheduled_groups %} -
    • - {{ group.acronym }} - {% if group.not_meeting %} - (Currently, this group does not plan to hold a session at IETF {{ meeting.number }}) - {% endif %} -
      • - {% empty %} -
    • NONE
      • - {% endfor %} -
    -
    -
    -
    -

    Edit / Cancel Previously Requested Sessions

    -

    The list below includes those working groups for which you or your co-chair has requested sessions at the upcoming IETF meeting. You can click on an acronym to initiate changes to a session, or cancel a session.

    - -
    -
    - -{% endblock %} - -{% block footer-extras %} - {% include "includes/sessions_footer.html" %} -{% endblock %} \ No newline at end of file diff --git a/ietf/secr/templates/sreq/new.html b/ietf/secr/templates/sreq/new.html deleted file mode 100755 index 3f46e6f897..0000000000 --- a/ietf/secr/templates/sreq/new.html +++ /dev/null @@ -1,43 +0,0 @@ -{% extends "base_site.html" %} -{% load static %} - -{% block title %}Sessions- New{% endblock %} - -{% block extrahead %}{{ block.super }} - - - {{ form.media }} - -{% endblock %} - -{% block breadcrumbs %}{{ block.super }} - » Sessions - » New Session Request -{% endblock %} - -{% block instructions %} - Instructions -{% endblock %} - -{% block content %} -
    -

    IETF {{ meeting.number }}: New Session Request

    -
    -
    -
      -
      • -
      • -
    -
    - {% include "includes/sessions_request_form.html" %} - -
    - -{% endblock %} - -{% block footer-extras %} - {% include "includes/sessions_footer.html" %} -{% endblock %} \ No newline at end of file diff --git a/ietf/secr/templates/sreq/tool_status.html b/ietf/secr/templates/sreq/tool_status.html deleted file mode 100755 index b91e73a129..0000000000 --- a/ietf/secr/templates/sreq/tool_status.html +++ /dev/null @@ -1,42 +0,0 @@ -{% extends "base_site.html" %} -{% load static %} - -{% block title %}Sessions{% endblock %} - -{% block extrahead %}{{ block.super }} - -{% endblock %} - -{% block breadcrumbs %}{{ block.super }} - » Sessions - » Session Status -{% endblock %} - -{% block content %} - -
    -

    Sessions - Status

    -

    Enter the message that you would like displayed to the WG Chair when this tool is locked.

    -
    {% csrf_token %} - - - - {{ form.as_table }} - -
    -
    -
      - {% if is_locked %} -
      • - {% else %} -
      • - {% endif %} -
      • -
    -
    - -
    - -
    - -{% endblock %} diff --git a/ietf/secr/templates/sreq/view.html b/ietf/secr/templates/sreq/view.html deleted file mode 100644 index 9a0a3b01c1..0000000000 --- a/ietf/secr/templates/sreq/view.html +++ /dev/null @@ -1,55 +0,0 @@ -{% extends "base_site.html" %} -{% load static %} - -{% block title %}Sessions - View{% endblock %} - -{% block extrahead %}{{ block.super }} - -{% endblock %} - -{% block extrastyle %} - -{% endblock %} - -{% block breadcrumbs %}{{ block.super }} - » Sessions - » {{ group.acronym }} -{% endblock %} - -{% block instructions %} - Instructions -{% endblock %} - -{% block content %} - -
    -

    Sessions - View (meeting: {{ meeting.number }})

    - - {% include "includes/sessions_request_view.html" %} - -
    - - {% include "includes/activities.html" %} - -
    -
      -
      • - {% if show_approve_button %} -
      • - {% endif %} -
      • -
      • -
    -
    -
    - -{% endblock %} - -{% block footer-extras %} - {% include "includes/sessions_footer.html" %} -{% endblock %} diff --git a/ietf/secr/urls.py b/ietf/secr/urls.py index 4a3e5b0363..ab21046654 100644 --- a/ietf/secr/urls.py +++ b/ietf/secr/urls.py @@ -1,11 +1,22 @@ +# Copyright The IETF Trust 2025, All Rights Reserved + +from django.conf import settings from django.urls import re_path, include from django.views.generic import TemplateView +from django.views.generic.base import RedirectView urlpatterns = [ re_path(r'^$', TemplateView.as_view(template_name='index.html'), name='ietf.secr'), re_path(r'^announcement/', include('ietf.secr.announcement.urls')), re_path(r'^meetings/', include('ietf.secr.meetings.urls')), re_path(r'^rolodex/', include('ietf.secr.rolodex.urls')), - re_path(r'^sreq/', include('ietf.secr.sreq.urls')), + # remove these redirects after 125 + re_path(r'^sreq/$', RedirectView.as_view(url='/meeting/session/request/', permanent=True)), + re_path(r'^sreq/%(acronym)s/$' % settings.URL_REGEXPS, RedirectView.as_view(url='/meeting/session/request/%(acronym)s/view/', permanent=True)), + re_path(r'^sreq/%(acronym)s/edit/$' % settings.URL_REGEXPS, RedirectView.as_view(url='/meeting/session/request/%(acronym)s/edit/', permanent=True)), + re_path(r'^sreq/%(acronym)s/new/$' % settings.URL_REGEXPS, RedirectView.as_view(url='/meeting/session/request/%(acronym)s/new/', permanent=True)), + re_path(r'^sreq/(?P[A-Za-z0-9_\-\+]+)/%(acronym)s/view/$' % settings.URL_REGEXPS, RedirectView.as_view(url='/meeting/%(num)s/session/request/%(acronym)s/view/', permanent=True)), + re_path(r'^sreq/(?P[A-Za-z0-9_\-\+]+)/%(acronym)s/edit/$' % settings.URL_REGEXPS, RedirectView.as_view(url='/meeting/%(num)s/session/request/%(acronym)s/edit/', permanent=True)), + # --------------------------------- re_path(r'^telechat/', include('ietf.secr.telechat.urls')), ] diff --git a/ietf/secr/utils/group.py b/ietf/secr/utils/group.py deleted file mode 100644 index 40a9065ace..0000000000 --- a/ietf/secr/utils/group.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright The IETF Trust 2013-2020, All Rights Reserved -# -*- coding: utf-8 -*- - - -# Python imports - -# Django imports -from django.core.exceptions import ObjectDoesNotExist - -# Datatracker imports -from ietf.group.models import Group -from ietf.ietfauth.utils import has_role - - -def get_my_groups(user,conclude=False): - ''' - Takes a Django user object (from request) - Returns a list of groups the user has access to. Rules are as follows - secretariat - has access to all groups - area director - has access to all groups in their area - wg chair or secretary - has access to their own group - chair of irtf has access to all irtf groups - - If user=None than all groups are returned. - concluded=True means include concluded groups. Need this to upload materials for groups - after they've been concluded. it happens. - ''' - my_groups = set() - states = ['bof','proposed','active'] - if conclude: - states.extend(['conclude','bof-conc']) - - all_groups = Group.objects.filter(type__features__has_meetings=True, state__in=states).order_by('acronym') - if user == None or has_role(user,'Secretariat'): - return all_groups - - try: - person = user.person - except ObjectDoesNotExist: - return list() - - for group in all_groups: - if group.role_set.filter(person=person,name__in=('chair','secr','ad')): - my_groups.add(group) - continue - if group.parent and group.parent.role_set.filter(person=person,name__in=('ad','chair')): - my_groups.add(group) - continue - - return list(my_groups) diff --git a/ietf/settings.py b/ietf/settings.py index 3af01d76e6..e0b4f20118 100644 --- a/ietf/settings.py +++ b/ietf/settings.py @@ -14,19 +14,29 @@ from hashlib import sha384 from typing import Any, Dict, List, Tuple # pyflakes:ignore +# DeprecationWarnings are suppressed by default, enable them warnings.simplefilter("always", DeprecationWarning) -warnings.filterwarnings("ignore", message="pkg_resources is deprecated as an API") -warnings.filterwarnings("ignore", "Log out via GET requests is deprecated") # happens in oidc_provider -warnings.filterwarnings("ignore", module="tastypie", message="The django.utils.datetime_safe module is deprecated.") -warnings.filterwarnings("ignore", module="oidc_provider", message="The django.utils.timezone.utc alias is deprecated.") + +# Warnings that must be resolved for Django 5.x +warnings.filterwarnings("ignore", "Log out via GET requests is deprecated") # caused by oidc_provider +warnings.filterwarnings("ignore", message="The django.utils.timezone.utc alias is deprecated.", module="oidc_provider") +warnings.filterwarnings("ignore", message="The django.utils.datetime_safe module is deprecated.", module="tastypie") warnings.filterwarnings("ignore", message="The USE_DEPRECATED_PYTZ setting,") # https://github.com/ietf-tools/datatracker/issues/5635 +warnings.filterwarnings("ignore", message="The is_dst argument to make_aware\\(\\)") # caused by django-filters when USE_DEPRECATED_PYTZ is true warnings.filterwarnings("ignore", message="The USE_L10N setting is deprecated.") # https://github.com/ietf-tools/datatracker/issues/5648 warnings.filterwarnings("ignore", message="django.contrib.auth.hashers.CryptPasswordHasher is deprecated.") # https://github.com/ietf-tools/datatracker/issues/5663 -warnings.filterwarnings("ignore", message="'urllib3\\[secure\\]' extra is deprecated") -warnings.filterwarnings("ignore", message="The logout\\(\\) view is superseded by") + +# Other DeprecationWarnings +warnings.filterwarnings("ignore", message="pkg_resources is deprecated as an API", module="pyang.plugin") warnings.filterwarnings("ignore", message="Report.file_reporters will no longer be available in Coverage.py 4.2", module="coverage.report") -warnings.filterwarnings("ignore", message="Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated", module="bleach") -warnings.filterwarnings("ignore", message="HTTPResponse.getheader\\(\\) is deprecated", module='selenium.webdriver') +warnings.filterwarnings("ignore", message="currentThread\\(\\) is deprecated", module="coverage.pytracer") +warnings.filterwarnings("ignore", message="co_lnotab is deprecated", module="coverage.parser") +warnings.filterwarnings("ignore", message="datetime.datetime.utcnow\\(\\) is deprecated", module="botocore.auth") +warnings.filterwarnings("ignore", message="datetime.datetime.utcnow\\(\\) is deprecated", module="oic.utils.time_util") +warnings.filterwarnings("ignore", message="datetime.datetime.utcfromtimestamp\\(\\) is deprecated", module="oic.utils.time_util") +warnings.filterwarnings("ignore", message="datetime.datetime.utcfromtimestamp\\(\\) is deprecated", module="pytz.tzinfo") +warnings.filterwarnings("ignore", message="'instantiateVariableFont' is deprecated", module="weasyprint") + base_path = pathlib.Path(__file__).resolve().parent BASE_DIR = str(base_path) @@ -216,6 +226,10 @@ BLOBSTORAGE_CONNECT_TIMEOUT = 10 # seconds; boto3 default is 60 BLOBSTORAGE_READ_TIMEOUT = 10 # seconds; boto3 default is 60 +# Caching for agenda data in seconds +AGENDA_CACHE_TIMEOUT_DEFAULT = 8 * 24 * 60 * 60 # 8 days +AGENDA_CACHE_TIMEOUT_CURRENT_MEETING = 6 * 60 # 6 minutes + WSGI_APPLICATION = "ietf.wsgi.application" AUTHENTICATION_BACKENDS = ( 'ietf.ietfauth.backends.CaseInsensitiveModelBackend', ) @@ -402,6 +416,7 @@ def skip_unreadable_post(record): ], 'OPTIONS': { 'context_processors': [ + 'ietf.context_processors.traceparent_id', 'django.contrib.auth.context_processors.auth', 'django.template.context_processors.debug', # makes 'sql_queries' available in templates 'django.template.context_processors.i18n', @@ -434,12 +449,14 @@ def skip_unreadable_post(record): MIDDLEWARE = [ + "ietf.middleware.add_otel_traceparent_header", "django.middleware.csrf.CsrfViewMiddleware", "corsheaders.middleware.CorsMiddleware", # see docs on CORS_REPLACE_HTTPS_REFERER before using it "django.middleware.common.CommonMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", + "ietf.middleware.is_authenticated_header_middleware", "django.middleware.http.ConditionalGetMiddleware", "simple_history.middleware.HistoryRequestMiddleware", # comment in this to get logging of SQL insert and update statements: @@ -447,23 +464,23 @@ def skip_unreadable_post(record): "ietf.middleware.SMTPExceptionMiddleware", "ietf.middleware.Utf8ExceptionMiddleware", "ietf.middleware.redirect_trailing_period_middleware", - "django_referrer_policy.middleware.ReferrerPolicyMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", "django.middleware.security.SecurityMiddleware", - #"csp.middleware.CSPMiddleware", "ietf.middleware.unicode_nfkc_normalization_middleware", - "ietf.middleware.is_authenticated_header_middleware", ] ROOT_URLCONF = 'ietf.urls' -DJANGO_VITE_ASSETS_PATH = os.path.join(BASE_DIR, 'static/dist-neue') +# Configure django_vite +DJANGO_VITE: dict = {"default": {}} if DEBUG: - DJANGO_VITE_MANIFEST_PATH = os.path.join(BASE_DIR, 'static/dist-neue/manifest.json') + DJANGO_VITE["default"]["manifest_path"] = os.path.join( + BASE_DIR, 'static/dist-neue/manifest.json' + ) # Additional locations of static files (in addition to each app's static/ dir) STATICFILES_DIRS = ( - DJANGO_VITE_ASSETS_PATH, + os.path.join(BASE_DIR, "static/dist-neue"), # for django_vite os.path.join(BASE_DIR, 'static/dist'), os.path.join(BASE_DIR, 'secr/static/dist'), ) @@ -488,6 +505,7 @@ def skip_unreadable_post(record): 'django_celery_results', 'corsheaders', 'django_markup', + 'django_filters', 'oidc_provider', 'drf_spectacular', 'drf_standardized_errors', @@ -527,7 +545,6 @@ def skip_unreadable_post(record): 'ietf.secr.announcement', 'ietf.secr.meetings', 'ietf.secr.rolodex', - 'ietf.secr.sreq', 'ietf.secr.telechat', ] @@ -567,8 +584,6 @@ def skip_unreadable_post(record): CORS_ALLOW_METHODS = ( 'GET', 'OPTIONS', ) CORS_URLS_REGEX = r'^(/api/.*|.*\.json|.*/json/?)$' -# Setting for django_referrer_policy.middleware.ReferrerPolicyMiddleware -REFERRER_POLICY = 'strict-origin-when-cross-origin' # django.middleware.security.SecurityMiddleware SECURE_BROWSER_XSS_FILTER = True @@ -581,6 +596,7 @@ def skip_unreadable_post(record): #SECURE_SSL_REDIRECT = True # Relax the COOP policy to allow Meetecho authentication pop-up SECURE_CROSS_ORIGIN_OPENER_POLICY = "unsafe-none" +SECURE_REFERRER_POLICY = "strict-origin-when-cross-origin" # Override this in your settings_local with the IP addresses relevant for you: INTERNAL_IPS = ( @@ -666,11 +682,6 @@ def skip_unreadable_post(record): IDNITS3_BASE_URL = "https://author-tools.ietf.org/idnits3/results" IDNITS_SERVICE_URL = "https://author-tools.ietf.org/idnits" -# Content security policy configuration (django-csp) -# (In current production, the Content-Security-Policy header is completely set by nginx configuration, but -# we try to keep this in sync to avoid confusion) -CSP_DEFAULT_SRC = ("'self'", "'unsafe-inline'", f"data: {IDTRACKER_BASE_URL} http://ietf.org/ https://www.ietf.org/ https://analytics.ietf.org/ https://static.ietf.org") - # The name of the method to use to invoke the test suite TEST_RUNNER = 'ietf.utils.test_runner.IetfTestRunner' @@ -709,6 +720,7 @@ def skip_unreadable_post(record): "ietf/utils/patch.py", "ietf/utils/test_data.py", "ietf/utils/jstest.py", + "ietf/utils/coverage.py", ] # These are code line regex patterns @@ -738,8 +750,8 @@ def skip_unreadable_post(record): TEST_CODE_COVERAGE_CHECKER = None if SERVER_MODE != 'production': - import coverage - TEST_CODE_COVERAGE_CHECKER = coverage.Coverage(source=[ BASE_DIR ], cover_pylib=False, omit=TEST_CODE_COVERAGE_EXCLUDE_FILES) + from ietf.utils.coverage import CoverageManager + TEST_CODE_COVERAGE_CHECKER = CoverageManager() TEST_CODE_COVERAGE_REPORT_PATH = "coverage/" TEST_CODE_COVERAGE_REPORT_URL = os.path.join(STATIC_URL, TEST_CODE_COVERAGE_REPORT_PATH, "index.html") @@ -782,29 +794,30 @@ def skip_unreadable_post(record): # Storages for artifacts stored as blobs ARTIFACT_STORAGE_NAMES: list[str] = [ - "bofreq", - "charter", - "conflrev", "active-draft", - "draft", - "slides", - "minutes", "agenda", + "bibxml-ids", "bluesheets", - "procmaterials", - "narrativeminutes", - "statement", - "statchg", - "liai-att", + "bofreq", + "charter", "chatlog", - "polls", - "staging", - "bibxml-ids", - "indexes", + "conflrev", + "draft", "floorplan", + "indexes", + "liai-att", "meetinghostlogo", + "minutes", + "narrativeminutes", "photo", + "polls", + "procmaterials", "review", + "rfc", + "slides", + "staging", + "statchg", + "statement", ] for storagename in ARTIFACT_STORAGE_NAMES: STORAGES[storagename] = { @@ -812,6 +825,25 @@ def skip_unreadable_post(record): "OPTIONS": {"bucket_name": storagename}, } +# Buckets / doc types of meeting materials the CF worker is allowed to serve. This +# differs from the list in Session.meeting_related() by the omission of "recording" +MATERIALS_TYPES_SERVED_BY_WORKER = [ + "agenda", + "bluesheets", + "chatlog", + "minutes", + "narrativeminutes", + "polls", + "procmaterials", + "slides", +] + +# Other storages +STORAGES["red_bucket"] = { + "BACKEND": "django.core.files.storage.InMemoryStorage", + "OPTIONS": {"location": "red_bucket"}, +} + # Override this in settings_local.py if needed # *_PATH variables ends with a slash/ . @@ -905,10 +937,11 @@ def skip_unreadable_post(record): RFC_EDITOR_QUEUE_URL = "https://www.rfc-editor.org/queue2.xml" RFC_EDITOR_INDEX_URL = "https://www.rfc-editor.org/rfc/rfc-index.xml" RFC_EDITOR_ERRATA_JSON_URL = "https://www.rfc-editor.org/errata.json" -RFC_EDITOR_ERRATA_URL = "https://www.rfc-editor.org/errata_search.php?rfc={rfc_number}" RFC_EDITOR_INLINE_ERRATA_URL = "https://www.rfc-editor.org/rfc/inline-errata/rfc{rfc_number}.html" +RFC_EDITOR_ERRATA_BASE_URL = "https://www.rfc-editor.org/errata/" RFC_EDITOR_INFO_BASE_URL = "https://www.rfc-editor.org/info/" + # NomCom Tool settings ROLODEX_URL = "" NOMCOM_PUBLIC_KEYS_DIR = '/a/www/nomcom/public_keys/' @@ -1341,6 +1374,11 @@ def skip_unreadable_post(record): MEETECHO_AUDIO_STREAM_URL = "https://mp3.conf.meetecho.com/ietf{session.meeting.number}/{session.pk}.m3u" MEETECHO_SESSION_RECORDING_URL = "https://meetecho-player.ietf.org/playout/?session={session_label}" +# Errata system api configuration +# settings should provide +# ERRATA_METADATA_NOTIFICATION_URL +# ERRATA_METADATA_NOTIFICATION_API_KEY + # Put the production SECRET_KEY in settings_local.py, and also any other # sensitive or site-specific changes. DO NOT commit settings_local.py to svn. from ietf.settings_local import * # pyflakes:ignore pylint: disable=wildcard-import @@ -1370,6 +1408,27 @@ def skip_unreadable_post(record): "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", "VERSION": __version__, "KEY_PREFIX": "ietf:dt", + # Key function is default except with sha384-encoded key + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, + "agenda": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt:agenda", + # Key function is default except with sha384-encoded key + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, + "proceedings": { + "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + "LOCATION": f"{MEMCACHED_HOST}:{MEMCACHED_PORT}", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt:proceedings", + # Key function is default except with sha384-encoded key "KEY_FUNCTION": lambda key, key_prefix, version: ( f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" ), @@ -1417,6 +1476,28 @@ def skip_unreadable_post(record): "VERSION": __version__, "KEY_PREFIX": "ietf:dt", }, + "agenda": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", + # "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + # "LOCATION": "127.0.0.1:11211", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt:agenda", + # Key function is default except with sha384-encoded key + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, + "proceedings": { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", + # "BACKEND": "ietf.utils.cache.LenientMemcacheCache", + # "LOCATION": "127.0.0.1:11211", + # No release-specific VERSION setting. + "KEY_PREFIX": "ietf:dt:proceedings", + # Key function is default except with sha384-encoded key + "KEY_FUNCTION": lambda key, key_prefix, version: ( + f"{key_prefix}:{version}:{sha384(str(key).encode('utf8')).hexdigest()}" + ), + }, "sessions": { "BACKEND": "django.core.cache.backends.locmem.LocMemCache", }, @@ -1472,11 +1553,17 @@ def skip_unreadable_post(record): NOMCOM_APP_SECRET = b'\x9b\xdas1\xec\xd5\xa0SI~\xcb\xd4\xf5t\x99\xc4i\xd7\x9f\x0b\xa9\xe8\xfeY\x80$\x1e\x12tN:\x84' ALLOWED_HOSTS = ['*',] - + try: # see https://github.com/omarish/django-cprofile-middleware - import django_cprofile_middleware # pyflakes:ignore - MIDDLEWARE = MIDDLEWARE + ['django_cprofile_middleware.middleware.ProfilerMiddleware', ] + import django_cprofile_middleware # pyflakes:ignore + + MIDDLEWARE = MIDDLEWARE + [ + "django_cprofile_middleware.middleware.ProfilerMiddleware", + ] + DJANGO_CPROFILE_MIDDLEWARE_REQUIRE_STAFF = ( + False # Do not use this setting for a public site! + ) except ImportError: pass @@ -1489,3 +1576,5 @@ def skip_unreadable_post(record): YOUTUBE_DOMAINS = ['www.youtube.com', 'youtube.com', 'youtu.be', 'm.youtube.com', 'youtube-nocookie.com', 'www.youtube-nocookie.com'] + +IETF_DOI_PREFIX = "10.17487" diff --git a/ietf/settings_test.py b/ietf/settings_test.py index 9a42e8b99d..1f5a7e8ddc 100755 --- a/ietf/settings_test.py +++ b/ietf/settings_test.py @@ -14,7 +14,7 @@ import shutil import tempfile from ietf.settings import * # pyflakes:ignore -from ietf.settings import TEST_CODE_COVERAGE_CHECKER, ORIG_AUTH_PASSWORD_VALIDATORS +from ietf.settings import ORIG_AUTH_PASSWORD_VALIDATORS, STORAGES import debug # pyflakes:ignore debug.debug = True @@ -52,10 +52,9 @@ def __getitem__(self, item): BLOBDB_DATABASE = "default" DATABASE_ROUTERS = [] # type: ignore -if TEST_CODE_COVERAGE_CHECKER and not TEST_CODE_COVERAGE_CHECKER._started: # pyflakes:ignore +if TEST_CODE_COVERAGE_CHECKER: # pyflakes:ignore TEST_CODE_COVERAGE_CHECKER.start() # pyflakes:ignore - def tempdir_with_cleanup(**kwargs): """Utility to create a temporary dir and arrange cleanup""" _dir = tempfile.mkdtemp(**kwargs) @@ -115,3 +114,9 @@ def tempdir_with_cleanup(**kwargs): AUTH_PASSWORD_VALIDATORS = ORIG_AUTH_PASSWORD_VALIDATORS except NameError: pass + +# Use InMemoryStorage for red bucket storage +STORAGES["red_bucket"] = { + "BACKEND": "django.core.files.storage.InMemoryStorage", + "OPTIONS": {"location": "red_bucket"}, +} diff --git a/ietf/settings_testcrawl.py b/ietf/settings_testcrawl.py index a1b5ce8946..edb978757a 100644 --- a/ietf/settings_testcrawl.py +++ b/ietf/settings_testcrawl.py @@ -27,9 +27,14 @@ 'MAX_ENTRIES': 10000, }, }, + 'agenda': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + }, + 'proceedings': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + }, 'sessions': { 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - # No version-specific VERSION setting. }, 'htmlized': { 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', diff --git a/ietf/static/css/datepicker.scss b/ietf/static/css/datepicker.scss index 88f9e835fd..b193ccda3a 100644 --- a/ietf/static/css/datepicker.scss +++ b/ietf/static/css/datepicker.scss @@ -4,3 +4,29 @@ $dp-cell-focus-background-color: $dropdown-link-hover-bg !default; @import "vanillajs-datepicker/sass/datepicker-bs5"; + +[data-bs-theme="dark"] .datepicker-picker { + .datepicker-header, + .datepicker-controls .btn, + .datepicker-main, + .datepicker-footer { + background-color: $gray-800; + } + + .datepicker-cell:hover { + background-color: $gray-700; + } + + .datepicker-cell.day.focused { + background-color: $gray-600; + } + + .datepicker-cell.day.selected.focused { + background-color: $blue; + } + + .datepicker-controls .btn:hover { + background-color:$gray-700; + color: $gray-400; + } +} diff --git a/ietf/static/js/custom_striped.js b/ietf/static/js/custom_striped.js new file mode 100644 index 0000000000..480ad7cf82 --- /dev/null +++ b/ietf/static/js/custom_striped.js @@ -0,0 +1,16 @@ +// Copyright The IETF Trust 2025, All Rights Reserved + +document.addEventListener('DOMContentLoaded', () => { + // add stripes + const firstRow = document.querySelector('.custom-stripe .row') + if (firstRow) { + const parent = firstRow.parentElement; + const allRows = Array.from(parent.children).filter(child => child.classList.contains('row')) + allRows.forEach((row, index) => { + row.classList.remove('bg-light') + if (index % 2 === 1) { + row.classList.add('bg-light') + } + }) + } +}) diff --git a/ietf/static/js/liaisons.js b/ietf/static/js/liaisons.js index e7a76660c9..6c46cb6dfc 100644 --- a/ietf/static/js/liaisons.js +++ b/ietf/static/js/liaisons.js @@ -197,16 +197,6 @@ var liaisonForm = { } }, - checkPostOnly: function (post_only) { - if (post_only) { - $("button[name=send]") - .hide(); - } else { - $("button[name=send]") - .show(); - } - }, - updateInfo: function (first_time, sender) { // don't overwrite fields when editing existing liaison if (liaisonForm.is_edit_form) { @@ -239,7 +229,6 @@ var liaisonForm = { liaisonForm.toggleApproval(response.needs_approval); liaisonForm.response_contacts.val(response.response_contacts); } - liaisonForm.checkPostOnly(response.post_only); } } }); @@ -326,4 +315,4 @@ $(document) .each(liaisonForm.init); $('#liaison_search_form') .each(searchForm.init); - }); \ No newline at end of file + }); diff --git a/ietf/static/js/list.js b/ietf/static/js/list.js index 756a75001a..c03368cd72 100644 --- a/ietf/static/js/list.js +++ b/ietf/static/js/list.js @@ -16,7 +16,7 @@ function text_sort(a, b, options) { // sort by text content return prep(a, options).localeCompare(prep(b, options), "en", { sensitivity: "base", - ignorePunctuation: true, + ignorePunctuation: false, numeric: true }); } diff --git a/ietf/static/js/session_details.js b/ietf/static/js/session_details.js new file mode 100644 index 0000000000..03d1b2d3d9 --- /dev/null +++ b/ietf/static/js/session_details.js @@ -0,0 +1,53 @@ +// Copyright The IETF Trust 2026, All Rights Reserved +// Relies on other scripts being loaded, see usage in session_details.html +document.addEventListener('DOMContentLoaded', () => { + // Init with best guess at local timezone. + ietf_timezone.set_tz_change_callback(timezone_changed) // cb is in upcoming.js + ietf_timezone.initialize('local') + + // Set up sortable elements if the user can manage materials + if (document.getElementById('can-manage-materials-flag')) { + const sortables = [] + const options = { + group: 'slides', + animation: 150, + handle: '.drag-handle', + onAdd: function (event) {onAdd(event)}, + onRemove: function (event) {onRemove(event)}, + onEnd: function (event) {onEnd(event)} + } + + function onAdd (event) { + const old_session = event.from.getAttribute('data-session') + const new_session = event.to.getAttribute('data-session') + $.post(event.to.getAttribute('data-add-to-session'), { + 'order': event.newIndex + 1, + 'name': event.item.getAttribute('name') + }) + $(event.item).find('td:eq(1)').find('a').each(function () { + $(this).attr('href', $(this).attr('href').replace(old_session, new_session)) + }) + } + + function onRemove (event) { + const old_session = event.from.getAttribute('data-session') + $.post(event.from.getAttribute('data-remove-from-session'), { + 'oldIndex': event.oldIndex + 1, + 'name': event.item.getAttribute('name') + }) + } + + function onEnd (event) { + if (event.to == event.from) { + $.post(event.from.getAttribute('data-reorder-in-session'), { + 'oldIndex': event.oldIndex + 1, + 'newIndex': event.newIndex + 1 + }) + } + } + + for (const elt of document.querySelectorAll('.slides tbody')) { + sortables.push(Sortable.create(elt, options)) + } + } +}) diff --git a/ietf/secr/static/js/session_form.js b/ietf/static/js/session_form.js similarity index 92% rename from ietf/secr/static/js/session_form.js rename to ietf/static/js/session_form.js index 6f28f16db4..bd61293d7c 100644 --- a/ietf/secr/static/js/session_form.js +++ b/ietf/static/js/session_form.js @@ -1,4 +1,4 @@ -/* Copyright The IETF Trust 2021, All Rights Reserved +/* Copyright The IETF Trust 2021-2025, All Rights Reserved * * JS support for the SessionForm * */ diff --git a/ietf/secr/static/js/sessions.js b/ietf/static/js/session_request.js similarity index 90% rename from ietf/secr/static/js/sessions.js rename to ietf/static/js/session_request.js index a2770e6262..dfb169f675 100644 --- a/ietf/secr/static/js/sessions.js +++ b/ietf/static/js/session_request.js @@ -1,4 +1,4 @@ -// Copyright The IETF Trust 2015-2021, All Rights Reserved +// Copyright The IETF Trust 2015-2025, All Rights Reserved /* global alert */ var ietf_sessions; // public interface @@ -38,7 +38,7 @@ var ietf_sessions; // public interface const only_one_session = (val === 1); if (document.form_post.session_time_relation) { document.form_post.session_time_relation.disabled = only_one_session; - document.form_post.session_time_relation.closest('tr').hidden = only_one_session; + document.form_post.session_time_relation.closest('div.row').hidden = only_one_session; } if (document.form_post.joint_for_session) { document.form_post.joint_for_session.disabled = only_one_session; @@ -129,6 +129,11 @@ var ietf_sessions; // public interface } } + function wg_constraint_delete_clicked(event) { + const constraint_name = event.currentTarget.dataset.constraint_name; + delete_last_wg_constraint(constraint_name); + } + /* Initialization */ function on_load() { // Attach event handler to session count select @@ -146,6 +151,9 @@ var ietf_sessions; // public interface selectors[index].addEventListener('change', wg_constraint_selector_changed, false) } + // Attach event handler to constraint delete buttons + document.querySelectorAll('.wg_constraint_delete') + .forEach(btn => btn.addEventListener('click', wg_constraint_delete_clicked)); } // initialize after page loads diff --git a/ietf/submit/checkers.py b/ietf/submit/checkers.py index 89908748a7..e02b686576 100644 --- a/ietf/submit/checkers.py +++ b/ietf/submit/checkers.py @@ -18,7 +18,7 @@ from ietf.utils import tool_version from ietf.utils.log import log, assertion from ietf.utils.pipe import pipe -from ietf.utils.test_runner import set_coverage_checking +from ietf.utils.test_runner import disable_coverage class DraftSubmissionChecker(object): name = "" @@ -247,34 +247,33 @@ def check_file_txt(self, path): ) # yanglint - set_coverage_checking(False) # we can't count the following as it may or may not be run, depending on setup - if settings.SUBMIT_YANGLINT_COMMAND and os.path.exists(settings.YANGLINT_BINARY): - cmd_template = settings.SUBMIT_YANGLINT_COMMAND - command = [ w for w in cmd_template.split() if not '=' in w ][0] - cmd = cmd_template.format(model=path, rfclib=settings.SUBMIT_YANG_RFC_MODEL_DIR, tmplib=workdir, - draftlib=settings.SUBMIT_YANG_DRAFT_MODEL_DIR, ianalib=settings.SUBMIT_YANG_IANA_MODEL_DIR, - cataloglib=settings.SUBMIT_YANG_CATALOG_MODEL_DIR, ) - code, out, err = pipe(cmd) - out = out.decode('utf-8') - err = err.decode('utf-8') - if code > 0 or len(err.strip()) > 0: - err_lines = err.splitlines() - for line in err_lines: - if line.strip(): - try: - if 'err : ' in line: - errors += 1 - if 'warn: ' in line: - warnings += 1 - except ValueError: - pass - #passed = passed and code == 0 # For the submission tool. Yang checks always pass - message += "{version}: {template}:\n{output}\n".format( - version=tool_version[command], - template=cmd_template, - output=out + "No validation errors\n" if (code == 0 and len(err) == 0) else out + err, - ) - set_coverage_checking(True) + with disable_coverage(): # pragma: no cover + if settings.SUBMIT_YANGLINT_COMMAND and os.path.exists(settings.YANGLINT_BINARY): + cmd_template = settings.SUBMIT_YANGLINT_COMMAND + command = [ w for w in cmd_template.split() if not '=' in w ][0] + cmd = cmd_template.format(model=path, rfclib=settings.SUBMIT_YANG_RFC_MODEL_DIR, tmplib=workdir, + draftlib=settings.SUBMIT_YANG_DRAFT_MODEL_DIR, ianalib=settings.SUBMIT_YANG_IANA_MODEL_DIR, + cataloglib=settings.SUBMIT_YANG_CATALOG_MODEL_DIR, ) + code, out, err = pipe(cmd) + out = out.decode('utf-8') + err = err.decode('utf-8') + if code > 0 or len(err.strip()) > 0: + err_lines = err.splitlines() + for line in err_lines: + if line.strip(): + try: + if 'err : ' in line: + errors += 1 + if 'warn: ' in line: + warnings += 1 + except ValueError: + pass + #passed = passed and code == 0 # For the submission tool. Yang checks always pass + message += "{version}: {template}:\n{output}\n".format( + version=tool_version[command], + template=cmd_template, + output=out + "No validation errors\n" if (code == 0 and len(err) == 0) else out + err, + ) else: errors += 1 message += "No such file: %s\nPossible mismatch between extracted xym file name and returned module name?\n" % (path) diff --git a/ietf/submit/tests.py b/ietf/submit/tests.py index 6b9002502b..400d0d8c7d 100644 --- a/ietf/submit/tests.py +++ b/ietf/submit/tests.py @@ -51,8 +51,9 @@ process_submission_xml, process_uploaded_submission, process_and_validate_submission, apply_yang_checker_to_draft, run_all_yang_model_checks) +from ietf.submit.views import access_token_is_valid, auth_token_is_valid from ietf.utils import tool_version -from ietf.utils.accesstoken import generate_access_token +from ietf.utils.accesstoken import generate_access_token, generate_random_key from ietf.utils.mail import outbox, get_payload_text from ietf.utils.test_runner import TestBlobstoreManager from ietf.utils.test_utils import login_testing_unauthorized, TestCase @@ -594,7 +595,7 @@ def submit_existing(self, formats, change_authors=True, group_type='wg', stream_ TestBlobstoreManager().emptyTestBlobstores() def _assert_authors_are_action_holders(draft, expect=True): - for author in draft.authors(): + for author in draft.author_persons(): if expect: self.assertIn(author, draft.action_holders.all()) else: @@ -2403,7 +2404,7 @@ def test_upload_draft(self): response = r.json() self.assertCountEqual( response.keys(), - ['id', 'name', 'rev', 'status_url'], + ['id', 'name', 'rev', 'status_url', 'submission_url'], ) submission_id = int(response['id']) self.assertEqual(response['name'], 'draft-somebody-test') @@ -2415,6 +2416,13 @@ def test_upload_draft(self): kwargs={'submission_id': submission_id}, ), ) + self.assertEqual( + response['submission_url'], + 'https://datatracker.example.com' + urlreverse( + 'ietf.submit.views.submission_status', + kwargs={'submission_id': submission_id}, + ) + ) self.assertEqual(mock_task.delay.call_count, 1) self.assertEqual(mock_task.delay.call_args.args, (submission_id,)) submission = Submission.objects.get(pk=submission_id) @@ -3500,3 +3508,31 @@ def test_submissionerror(self, mock_sanitize_message): mock_sanitize_message.call_args_list, [mock.call("hi"), mock.call("there")], ) + + +class HelperTests(TestCase): + def test_access_token_is_valid(self): + submission: Submission = SubmissionFactory() # type: ignore + valid_token = submission.access_token() + access_key = submission.access_key # accept this for backwards compat + invalid_token = "not the valid token" + self.assertTrue(access_token_is_valid(submission, valid_token)) + self.assertTrue(access_token_is_valid(submission, access_key)) + self.assertFalse(access_token_is_valid(submission, invalid_token)) + + def test_auth_token_is_valid(self): + auth_key = generate_random_key() + submission: Submission = SubmissionFactory(auth_key = auth_key) # type: ignore + valid_token = generate_access_token(submission.auth_key) + auth_key = submission.auth_key # accept this for backwards compat + invalid_token = "not the valid token" + self.assertTrue(auth_token_is_valid(submission, valid_token)) + self.assertTrue(auth_token_is_valid(submission, auth_key)) + self.assertFalse(auth_token_is_valid(submission, invalid_token)) + + submission.auth_key = "" + submission.save() + self.assertFalse(auth_token_is_valid(submission, valid_token)) + self.assertFalse(auth_token_is_valid(submission, auth_key)) + self.assertFalse(auth_token_is_valid(submission, invalid_token)) + self.assertFalse(auth_token_is_valid(submission, "")) diff --git a/ietf/submit/utils.py b/ietf/submit/utils.py index a0c7dd8511..9a7c358a6d 100644 --- a/ietf/submit/utils.py +++ b/ietf/submit/utils.py @@ -1268,7 +1268,7 @@ def process_submission_text(filename, revision): if title: title = _normalize_title(title) - # Translation taable drops \r, \n, <, >. + # Translation table drops \r, \n, <, >. trans_table = str.maketrans("", "", "\r\n<>") authors = [ { diff --git a/ietf/submit/views.py b/ietf/submit/views.py index 043b613016..2db3f51098 100644 --- a/ietf/submit/views.py +++ b/ietf/submit/views.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- import re import datetime +from secrets import compare_digest from typing import Optional, cast # pyflakes:ignore from urllib.parse import urljoin @@ -181,6 +182,10 @@ def err(code, error, messages=None): settings.IDTRACKER_BASE_URL, urlreverse(api_submission_status, kwargs={'submission_id': submission.pk}), ), + 'submission_url': urljoin( + settings.IDTRACKER_BASE_URL, + urlreverse("ietf.submit.views.submission_status", kwargs={'submission_id': submission.pk}), + ), } ) else: @@ -255,19 +260,48 @@ def search_submission(request): ) -def can_edit_submission(user, submission, access_token): - key_matched = access_token and submission.access_token() == access_token - if not key_matched: key_matched = submission.access_key == access_token # backwards-compat - return key_matched or has_role(user, "Secretariat") +def access_token_is_valid(submission: Submission, access_token: str): + """Check whether access_token is valid for submission, in constant time""" + token_matched = compare_digest(submission.access_token(), access_token) + # also compare key directly for backwards compatibility + key_matched = compare_digest(submission.access_key, access_token) + return token_matched or key_matched + + +def auth_token_is_valid(submission: Submission, auth_token: str): + """Check whether auth_token is valid for submission, in constant time""" + auth_key = submission.auth_key + if not auth_key: + # Make the same calls as the other branch to keep constant time, then + # return False because there is no auth key + compare_digest(generate_access_token("fake"), auth_token) + compare_digest("fake", auth_token) + return False + else: + token_matched = compare_digest(generate_access_token(auth_key), auth_token) + # also compare key directly for backwards compatibility + key_matched = compare_digest(auth_key, auth_token) + return token_matched or key_matched + + +def can_edit_submission(user, submission: Submission, access_token: str | None): + if has_role(user, "Secretariat"): + return True + elif not access_token: + return False + return access_token_is_valid(submission, access_token) + def submission_status(request, submission_id, access_token=None): # type: (HttpRequest, str, Optional[str]) -> HttpResponse submission = get_object_or_404(Submission, pk=submission_id) - key_matched = access_token and submission.access_token() == access_token - if not key_matched: key_matched = submission.access_key == access_token # backwards-compat - if access_token and not key_matched: - raise Http404 + if access_token: + key_matched = access_token_is_valid(submission, access_token) + if not key_matched: + raise Http404 + else: + key_matched = False if submission.state.slug == "cancel": errors = {} @@ -621,8 +655,7 @@ def edit_submission(request, submission_id, access_token=None): def confirm_submission(request, submission_id, auth_token): submission = get_object_or_404(Submission, pk=submission_id) - key_matched = submission.auth_key and auth_token == generate_access_token(submission.auth_key) - if not key_matched: key_matched = auth_token == submission.auth_key # backwards-compat + key_matched = submission.auth_key and auth_token_is_valid(submission, auth_token) if request.method == 'POST' and submission.state_id in ("auth", "aut-appr") and key_matched: # Set a temporary state 'confirmed' to avoid entering this code diff --git a/ietf/sync/iana.py b/ietf/sync/iana.py index f46fe407d4..0d40c5337e 100644 --- a/ietf/sync/iana.py +++ b/ietf/sync/iana.py @@ -66,8 +66,8 @@ def update_rfc_log_from_protocol_page(rfc_names, rfc_must_published_later_than): def fetch_changes_json(url, start, end): - url += "?start=%s&end=%s" % (urlquote(start.astimezone(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S")), - urlquote(end.astimezone(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S"))) + url += "?start=%s&end=%s" % (urlquote(start.astimezone(datetime.UTC).strftime("%Y-%m-%d %H:%M:%S")), + urlquote(end.astimezone(datetime.UTC).strftime("%Y-%m-%d %H:%M:%S"))) # HTTP basic auth username = "ietfsync" password = settings.IANA_SYNC_PASSWORD @@ -161,7 +161,7 @@ def update_history_with_changes(changes, send_email=True): for c in changes: docname = c['doc'] - timestamp = datetime.datetime.strptime(c["time"], "%Y-%m-%d %H:%M:%S",).replace(tzinfo=datetime.timezone.utc) + timestamp = datetime.datetime.strptime(c["time"], "%Y-%m-%d %H:%M:%S",).replace(tzinfo=datetime.UTC) if c['type'] in ("iana_state", "iana_review"): if c['type'] == "iana_state": @@ -247,7 +247,7 @@ def parse_review_email(text): review_time = parsedate_to_datetime(msg["Date"]) # parsedate_to_datetime() may return a naive timezone - treat as UTC if review_time.tzinfo is None or review_time.tzinfo.utcoffset(review_time) is None: - review_time = review_time.replace(tzinfo=datetime.timezone.utc) + review_time = review_time.replace(tzinfo=datetime.UTC) # by by = None diff --git a/ietf/sync/rfceditor.py b/ietf/sync/rfceditor.py index 76357ffbcc..aa0e643b20 100644 --- a/ietf/sync/rfceditor.py +++ b/ietf/sync/rfceditor.py @@ -115,6 +115,8 @@ def parse_queue(response): stream = "irtf" elif name.startswith("INDEPENDENT"): stream = "ise" + elif name.startswith("Editorial Stream"): + stream = "editorial" else: stream = None warnings.append("unrecognized section " + name) @@ -466,14 +468,18 @@ def update_docs_from_rfc_index( doc.set_state(rfc_published_state) if draft: doc.formal_languages.set(draft.formal_languages.all()) - for author in draft.documentauthor_set.all(): + # Create authors based on the last draft in the datatracker. This + # path will go away when we publish via the modernized RPC workflow + # but until then, these are the only data we have for authors that + # are easily connected to Person records. + for documentauthor in draft.documentauthor_set.all(): # Copy the author but point at the new doc. # See https://docs.djangoproject.com/en/4.2/topics/db/queries/#copying-model-instances - author.pk = None - author.id = None - author._state.adding = True - author.document = doc - author.save() + documentauthor.pk = None + documentauthor.id = None + documentauthor._state.adding = True + documentauthor.document = doc + documentauthor.save() if draft: draft_events = [] @@ -630,43 +636,70 @@ def update_docs_from_rfc_index( ) rfc_published = True - def parse_relation_list(l): - res = [] - for x in l: - for a in Document.objects.filter(name=x.lower(), type_id="rfc"): - if a not in res: - res.append(a) - return res + def parse_relation_list(rel_list: list[str]) -> list[Document]: + return list( + Document.objects.filter( + name__in=[name.strip().lower() for name in rel_list], + type_id="rfc" + ) + ) - for x in parse_relation_list(obsoletes): - if not RelatedDocument.objects.filter( - source=doc, target=x, relationship=relationship_obsoletes + # Create missing obsoletes relations + docs_this_obsoletes = parse_relation_list(obsoletes) + for obs_doc in docs_this_obsoletes: + if not doc.relateddocument_set.filter( + target=obs_doc, relationship=relationship_obsoletes ): - r = RelatedDocument.objects.create( - source=doc, target=x, relationship=relationship_obsoletes + r = doc.relateddocument_set.create( + target=obs_doc, relationship=relationship_obsoletes ) rfc_changes.append( - "created {rel_name} relation between {src_name} and {tgt_name}".format( + "created {rel_name} relation between {src} and {tgt}".format( rel_name=r.relationship.name.lower(), - src_name=prettify_std_name(r.source.name), - tgt_name=prettify_std_name(r.target.name), + src=prettify_std_name(r.source.name), + tgt=prettify_std_name(r.target.name), ) ) + # Remove stale obsoletes relations + for r in doc.relateddocument_set.filter( + relationship=relationship_obsoletes + ).exclude(target_id__in=[d.pk for d in docs_this_obsoletes]): + r.delete() + rfc_changes.append( + "removed {rel_name} relation between {src} and {tgt}".format( + rel_name=r.relationship.name.lower(), + src=prettify_std_name(r.source.name), + tgt=prettify_std_name(r.target.name), + ) + ) - for x in parse_relation_list(updates): + docs_this_updates = parse_relation_list(updates) + for upd_doc in docs_this_updates: if not RelatedDocument.objects.filter( - source=doc, target=x, relationship=relationship_updates + source=doc, target=upd_doc, relationship=relationship_updates ): - r = RelatedDocument.objects.create( - source=doc, target=x, relationship=relationship_updates + r = doc.relateddocument_set.create( + target=upd_doc, relationship=relationship_updates ) rfc_changes.append( - "created {rel_name} relation between {src_name} and {tgt_name}".format( + "created {rel_name} relation between {src} and {tgt}".format( rel_name=r.relationship.name.lower(), - src_name=prettify_std_name(r.source.name), - tgt_name=prettify_std_name(r.target.name), + src=prettify_std_name(r.source.name), + tgt=prettify_std_name(r.target.name), ) ) + # Remove stale updates relations + for r in doc.relateddocument_set.filter( + relationship=relationship_updates + ).exclude(target_id__in=[d.pk for d in docs_this_updates]): + r.delete() + rfc_changes.append( + "removed {rel_name} relation between {src} and {tgt}".format( + rel_name=r.relationship.name.lower(), + src=prettify_std_name(r.source.name), + tgt=prettify_std_name(r.target.name), + ) + ) if also: # recondition also to have proper subseries document names: diff --git a/ietf/sync/rfcindex.py b/ietf/sync/rfcindex.py new file mode 100644 index 0000000000..63c2044931 --- /dev/null +++ b/ietf/sync/rfcindex.py @@ -0,0 +1,483 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +import json +from collections import defaultdict +from collections.abc import Container +from dataclasses import dataclass +from itertools import chain +from operator import attrgetter, itemgetter +from pathlib import Path +from textwrap import fill +from urllib.parse import urljoin + +from django.conf import settings +from django.core.files.base import ContentFile +from lxml import etree + +from django.core.files.storage import storages +from django.db import models +from django.db.models.functions import Substr, Cast +from django.template.loader import render_to_string +from django.utils import timezone + +from ietf.doc.models import Document +from ietf.name.models import StdLevelName +from ietf.utils.log import log + +FORMATS_FOR_INDEX = ["txt", "html", "pdf", "xml", "ps"] + + +def format_rfc_number(n): + """Format an RFC number (or subseries doc number) + + Set settings.RFCINDEX_MATCH_LEGACY_XML=True for the legacy (leading-zero) format. + That is for debugging only - tests will fail. + """ + if getattr(settings, "RFCINDEX_MATCH_LEGACY_XML", False): + return format(n, "04") + else: + return format(n) + + +def errata_url(rfc: Document): + return urljoin(settings.RFC_EDITOR_ERRATA_BASE_URL + "/", f"rfc{rfc.rfc_number}") + + +def save_to_red_bucket(filename: str, content: str | bytes): + red_bucket = storages["red_bucket"] + bucket_path = str(Path(getattr(settings, "RFCINDEX_OUTPUT_PATH", "")) / filename) + if getattr(settings, "RFCINDEX_DELETE_THEN_WRITE", True): + # Django 4.2's FileSystemStorage does not support allow_overwrite. + red_bucket.delete(bucket_path) + red_bucket.save( + bucket_path, + ContentFile(content if isinstance(content, bytes) else content.encode("utf-8")), + ) + log(f"Saved {bucket_path} in red_bucket storage") + + +@dataclass +class UnusableRfcNumber: + rfc_number: int + comment: str + + +def get_unusable_rfc_numbers() -> list[UnusableRfcNumber]: + FILENAME = "unusable-rfc-numbers.json" + bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME) + try: + with storages["red_bucket"].open(bucket_path) as urn_file: + records = json.load(urn_file) + except FileNotFoundError: + if settings.SERVER_MODE == "development": + log( + f"Unable to open {bucket_path} in red_bucket storage. This is okay in dev " + "but generated rfc-index will not agree with RFC Editor values." + ) # pragma: no cover + return [] # pragma: no cover + log(f"Error: unable to open {bucket_path} in red_bucket storage") + raise + except json.JSONDecodeError: + log(f"Error: unable to parse {bucket_path} in red_bucket storage") + if settings.SERVER_MODE == "development": + return [] # pragma: no cover + raise + assert all(isinstance(record["number"], int) for record in records) + assert all(isinstance(record["comment"], str) for record in records) + return [ + UnusableRfcNumber(rfc_number=record["number"], comment=record["comment"]) + for record in sorted(records, key=itemgetter("number")) + ] + + +def get_april1_rfc_numbers() -> Container[int]: + FILENAME = "april-first-rfc-numbers.json" + bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME) + try: + with storages["red_bucket"].open(bucket_path) as urn_file: + records = json.load(urn_file) + except FileNotFoundError: + if settings.SERVER_MODE == "development": + log( + f"Unable to open {bucket_path} in red_bucket storage. This is okay in dev " + "but generated rfc-index will not agree with RFC Editor values." + ) # pragma: no cover + return [] # pragma: no cover + log(f"Error: unable to open {bucket_path} in red_bucket storage") + raise + except json.JSONDecodeError: + log(f"Error: unable to parse {bucket_path} in red_bucket storage") + if settings.SERVER_MODE == "development": + return [] # pragma: no cover + raise + assert all(isinstance(record, int) for record in records) + return records + + +def get_publication_std_levels() -> dict[int, StdLevelName]: + FILENAME = "publication-std-levels.json" + bucket_path = str(Path(getattr(settings, "RFCINDEX_INPUT_PATH", "")) / FILENAME) + values: dict[int, StdLevelName] = {} + try: + with storages["red_bucket"].open(bucket_path) as urn_file: + records = json.load(urn_file) + except FileNotFoundError: + if settings.SERVER_MODE == "development": + log( + f"Unable to open {bucket_path} in red_bucket storage. This is okay in dev " + "but generated rfc-index will not agree with RFC Editor values." + ) # pragma: no cover + # intentionally fall through instead of return here + else: + log(f"Error: unable to open {bucket_path} in red_bucket storage") + raise + except json.JSONDecodeError: + log(f"Error: unable to parse {bucket_path} in red_bucket storage") + if settings.SERVER_MODE != "development": + raise + else: + assert all(isinstance(record["number"], int) for record in records) + values = { + record["number"]: StdLevelName.objects.get( + slug=record["publication_std_level"] + ) + for record in records + } + # defaultdict to return "unknown" for any missing values + unknown_std_level = StdLevelName.objects.get(slug="unkn") + return defaultdict(lambda: unknown_std_level, values) + + +def format_ordering(rfc_number): + if rfc_number < 8650: + ordering = ["txt", "ps", "pdf", "html", "xml"] + else: + ordering = ["html", "txt", "ps", "pdf", "xml"] + return ordering.index # return the method + + +def get_rfc_text_index_entries(): + """Returns RFC entries for rfc-index.txt""" + entries = [] + april1_rfc_numbers = get_april1_rfc_numbers() + published_rfcs = Document.objects.filter(type_id="rfc").order_by("rfc_number") + rfcs = sorted( + chain(published_rfcs, get_unusable_rfc_numbers()), key=attrgetter("rfc_number") + ) + for rfc in rfcs: + if isinstance(rfc, UnusableRfcNumber): + entries.append(f"{format_rfc_number(rfc.rfc_number)} Not Issued.") + else: + assert isinstance(rfc, Document) + authors = ", ".join( + author.format_for_titlepage() for author in rfc.rfcauthor_set.all() + ) + published_at = rfc.pub_date() + date = ( + published_at.strftime("1 %B %Y") + if rfc.rfc_number in april1_rfc_numbers + else published_at.strftime("%B %Y") + ) + + # formats + formats = ", ".join( + sorted( + [ + format["fmt"] + for format in rfc.formats() + if format["fmt"] in FORMATS_FOR_INDEX + ], + key=format_ordering(rfc.rfc_number), + ) + ).upper() + + # obsoletes + obsoletes = "" + obsoletes_documents = sorted( + rfc.related_that_doc("obs"), + key=attrgetter("rfc_number"), + ) + if len(obsoletes_documents) > 0: + obsoletes_names = ", ".join( + f"RFC{format_rfc_number(doc.rfc_number)}" + for doc in obsoletes_documents + ) + obsoletes = f" (Obsoletes {obsoletes_names})" + + # obsoleted by + obsoleted_by = "" + obsoleted_by_documents = sorted( + rfc.related_that("obs"), + key=attrgetter("rfc_number"), + ) + if len(obsoleted_by_documents) > 0: + obsoleted_by_names = ", ".join( + f"RFC{format_rfc_number(doc.rfc_number)}" + for doc in obsoleted_by_documents + ) + obsoleted_by = f" (Obsoleted by {obsoleted_by_names})" + + # updates + updates = "" + updates_documents = sorted( + rfc.related_that_doc("updates"), + key=attrgetter("rfc_number"), + ) + if len(updates_documents) > 0: + updates_names = ", ".join( + f"RFC{format_rfc_number(doc.rfc_number)}" + for doc in updates_documents + ) + updates = f" (Updates {updates_names})" + + # updated by + updated_by = "" + updated_by_documents = sorted( + rfc.related_that("updates"), + key=attrgetter("rfc_number"), + ) + if len(updated_by_documents) > 0: + updated_by_names = ", ".join( + f"RFC{format_rfc_number(doc.rfc_number)}" + for doc in updated_by_documents + ) + updated_by = f" (Updated by {updated_by_names})" + + doc_relations = f"{obsoletes}{obsoleted_by}{updates}{updated_by} " + + # subseries + subseries = ",".join( + f"{container.type.slug}{format_rfc_number(int(container.name[3:]))}" + for container in rfc.part_of() + ).upper() + if subseries: + subseries = f"(Also {subseries}) " + + entry = fill( + ( + f"{format_rfc_number(rfc.rfc_number)} {rfc.title}. {authors}. {date}. " + f"(Format: {formats}){doc_relations}{subseries}" + f"(Status: {str(rfc.std_level).upper()}) " + f"(DOI: {rfc.doi})" + ), + width=73, + subsequent_indent=" " * 5, + ) + entries.append(entry) + + return entries + + +def add_subseries_xml_index_entries(rfc_index, ss_type, include_all=False): + """Add subseries entries for rfc-index.xml""" + # subseries docs annotated with numeric number + ss_docs = list( + Document.objects.filter(type_id=ss_type) + .annotate( + number=Cast( + Substr("name", 4, None), + output_field=models.IntegerField(), + ) + ) + .order_by("-number") + ) + if len(ss_docs) == 0: + return # very much not expected + highest_number = ss_docs[0].number + for ss_number in range(1, highest_number + 1): + if ss_docs[-1].number == ss_number: + this_ss_doc = ss_docs.pop() + contained_rfcs = this_ss_doc.contains() + else: + contained_rfcs = [] + if len(contained_rfcs) == 0 and not include_all: + continue + entry = etree.SubElement(rfc_index, f"{ss_type}-entry") + etree.SubElement( + entry, "doc-id" + ).text = f"{ss_type.upper()}{format_rfc_number(ss_number)}" + if len(contained_rfcs) > 0: + is_also = etree.SubElement(entry, "is-also") + for rfc in sorted(contained_rfcs, key=attrgetter("rfc_number")): + etree.SubElement( + is_also, "doc-id" + ).text = f"RFC{format_rfc_number(rfc.rfc_number)}" + + +def add_related_xml_index_entries(root: etree.Element, rfc: Document, tag: str): + relation_getter = { + "obsoletes": lambda doc: doc.related_that_doc("obs"), + "obsoleted-by": lambda doc: doc.related_that("obs"), + "updates": lambda doc: doc.related_that_doc("updates"), + "updated-by": lambda doc: doc.related_that("updates"), + } + related_docs = sorted( + relation_getter[tag](rfc), + key=attrgetter("rfc_number"), + ) + if len(related_docs) > 0: + element = etree.SubElement(root, tag) + for doc in related_docs: + etree.SubElement( + element, "doc-id" + ).text = f"RFC{format_rfc_number(doc.rfc_number)}" + + +def add_rfc_xml_index_entries(rfc_index): + """Add RFC entries for rfc-index.xml""" + entries = [] + april1_rfc_numbers = get_april1_rfc_numbers() + publication_statuses = get_publication_std_levels() + + published_rfcs = Document.objects.filter(type_id="rfc").order_by("rfc_number") + + # Iterators for unpublished and published, both sorted by number + unpublished_iter = iter(get_unusable_rfc_numbers()) + published_iter = iter(published_rfcs) + + # Prime the next_* values + next_unpublished = next(unpublished_iter, None) + next_published = next(published_iter, None) + + while next_published is not None or next_unpublished is not None: + if next_unpublished is not None and ( + next_published is None + or next_unpublished.rfc_number < next_published.rfc_number + ): + entry = etree.SubElement(rfc_index, "rfc-not-issued-entry") + etree.SubElement( + entry, "doc-id" + ).text = f"RFC{format_rfc_number(next_unpublished.rfc_number)}" + entries.append(entry) + next_unpublished = next(unpublished_iter, None) + continue + + rfc = next_published # hang on to this + next_published = next(published_iter, None) # prep for next iteration + entry = etree.SubElement(rfc_index, "rfc-entry") + + etree.SubElement( + entry, "doc-id" + ).text = f"RFC{format_rfc_number(rfc.rfc_number)}" + etree.SubElement(entry, "title").text = rfc.title + + for author in rfc.rfcauthor_set.all(): + author_element = etree.SubElement(entry, "author") + etree.SubElement(author_element, "name").text = author.titlepage_name + if author.is_editor: + etree.SubElement(author_element, "title").text = "Editor" + + date = etree.SubElement(entry, "date") + published_at = rfc.pub_date() + etree.SubElement(date, "month").text = published_at.strftime("%B") + if rfc.rfc_number in april1_rfc_numbers: + etree.SubElement(date, "day").text = str(published_at.day) + etree.SubElement(date, "year").text = str(published_at.year) + + format_ = etree.SubElement(entry, "format") + fmts = [ff["fmt"] for ff in rfc.formats() if ff["fmt"] in FORMATS_FOR_INDEX] + for fmt in sorted(fmts, key=format_ordering(rfc.rfc_number)): + match_legacy = getattr(settings, "RFCINDEX_MATCH_LEGACY_XML", False) + etree.SubElement(format_, "file-format").text = ( + "ASCII" if match_legacy and fmt == "txt" else fmt.upper() + ) + + etree.SubElement(entry, "page-count").text = str(rfc.pages) + + if len(rfc.keywords) > 0: + keywords = etree.SubElement(entry, "keywords") + for keyword in rfc.keywords: + etree.SubElement(keywords, "kw").text = keyword.strip() + + if rfc.abstract: + abstract = etree.SubElement(entry, "abstract") + for paragraph in rfc.abstract.split("\n\n"): + etree.SubElement(abstract, "p").text = paragraph.strip() + + draft = rfc.came_from_draft() + if draft is not None: + etree.SubElement(entry, "draft").text = f"{draft.name}-{draft.rev}" + + part_of_documents = rfc.part_of() + if len(part_of_documents) > 0: + is_also = etree.SubElement(entry, "is-also") + for doc in part_of_documents: + etree.SubElement(is_also, "doc-id").text = doc.name.upper() + + add_related_xml_index_entries(entry, rfc, "obsoletes") + add_related_xml_index_entries(entry, rfc, "obsoleted-by") + add_related_xml_index_entries(entry, rfc, "updates") + add_related_xml_index_entries(entry, rfc, "updated-by") + + etree.SubElement(entry, "current-status").text = rfc.std_level.name.upper() + etree.SubElement(entry, "publication-status").text = publication_statuses[ + rfc.rfc_number + ].name.upper() + etree.SubElement(entry, "stream").text = ( + "INDEPENDENT" if rfc.stream_id == "ise" else rfc.stream.name + ) + + # Add area / wg_acronym + if rfc.stream_id == "ietf": + if rfc.group.type_id in ["individ", "area"]: + etree.SubElement(entry, "wg_acronym").text = "NON WORKING GROUP" + else: + if rfc.area is not None: + etree.SubElement(entry, "area").text = rfc.area.acronym + if rfc.group: + etree.SubElement(entry, "wg_acronym").text = rfc.group.acronym + + if rfc.tags.filter(slug="errata").exists(): + etree.SubElement(entry, "errata-url").text = errata_url(rfc) + etree.SubElement(entry, "doi").text = rfc.doi + entries.append(entry) + + +def create_rfc_txt_index(): + """Create text index of published documents""" + DATE_FMT = "%m/%d/%Y" + created_on = timezone.now().strftime(DATE_FMT) + log("Creating rfc-index.txt") + index = render_to_string( + "sync/rfc-index.txt", + { + "created_on": created_on, + "rfcs": get_rfc_text_index_entries(), + }, + ) + save_to_red_bucket("rfc-index.txt", index) + + +def create_rfc_xml_index(): + """Create XML index of published documents""" + XSI_NAMESPACE = "http://www.w3.org/2001/XMLSchema-instance" + XSI = "{" + XSI_NAMESPACE + "}" + + log("Creating rfc-index.xml") + rfc_index = etree.Element( + "rfc-index", + nsmap={ + None: "https://www.rfc-editor.org/rfc-index", + "xsi": XSI_NAMESPACE, + }, + attrib={ + XSI + "schemaLocation": ( + "https://www.rfc-editor.org/rfc-index " + "https://www.rfc-editor.org/rfc-index.xsd" + ), + }, + ) + + # add data + add_subseries_xml_index_entries(rfc_index, "bcp", include_all=True) + add_subseries_xml_index_entries(rfc_index, "fyi") + add_rfc_xml_index_entries(rfc_index) + add_subseries_xml_index_entries(rfc_index, "std") + + # make it pretty + pretty_index = etree.tostring( + rfc_index, + encoding="utf-8", + xml_declaration=True, + pretty_print=4, + ) + save_to_red_bucket("rfc-index.xml", pretty_index) diff --git a/ietf/sync/tasks.py b/ietf/sync/tasks.py index 18ab4fe66e..4c84dc581e 100644 --- a/ietf/sync/tasks.py +++ b/ietf/sync/tasks.py @@ -1,9 +1,11 @@ -# Copyright The IETF Trust 2024, All Rights Reserved +# Copyright The IETF Trust 2024-2026, All Rights Reserved # # Celery task definitions # import datetime import io +from pathlib import Path +from tempfile import NamedTemporaryFile import requests from celery import shared_task @@ -12,9 +14,12 @@ from django.utils import timezone from ietf.doc.models import DocEvent, RelatedDocument +from ietf.doc.tasks import rebuild_reference_relations_task from ietf.sync import iana from ietf.sync import rfceditor from ietf.sync.rfceditor import MIN_QUEUE_RESULTS, parse_queue, update_drafts_from_queue +from ietf.sync.rfcindex import create_rfc_txt_index, create_rfc_xml_index +from ietf.sync.utils import build_from_file_content, load_rfcs_into_blobdb, rsync_helper from ietf.utils import log from ietf.utils.timezone import date_today @@ -65,11 +70,16 @@ def rfc_editor_index_update_task(full_index=False): if len(errata_data) < rfceditor.MIN_ERRATA_RESULTS: log.log("Not enough errata entries, only %s" % len(errata_data)) return # failed + newly_published = set() for rfc_number, changes, doc, rfc_published in rfceditor.update_docs_from_rfc_index( index_data, errata_data, skip_older_than_date=skip_date ): for c in changes: log.log("RFC%s, %s: %s" % (rfc_number, doc.name, c)) + if rfc_published: + newly_published.add(rfc_number) + if len(newly_published) > 0: + rsync_rfcs_from_rfceditor_task.delay(list(newly_published)) @shared_task @@ -152,7 +162,7 @@ def iana_protocols_update_task(): 2012, 11, 26, - tzinfo=datetime.timezone.utc, + tzinfo=datetime.UTC, ) try: @@ -222,3 +232,51 @@ def fix_subseries_docevents_task(): DocEvent.objects.filter(type="sync_from_rfc_editor", desc=desc).update( time=obsoleting_time ) + +@shared_task +def rsync_rfcs_from_rfceditor_task(rfc_numbers: list[int]): + log.log(f"Rsyncing rfcs from rfc-editor: {rfc_numbers}") + from_file = None + with NamedTemporaryFile(mode="w", delete_on_close=False) as fp: + fp.write(build_from_file_content(rfc_numbers)) + fp.close() + from_file = Path(fp.name) + rsync_helper( + [ + "-a", + "--ignore-existing", + f"--include-from={from_file}", + "--exclude=*", + "rsync.rfc-editor.org::rfcs/", + f"{settings.RFC_PATH}", + ] + ) + load_rfcs_into_blobdb(rfc_numbers) + + rebuild_reference_relations_task.delay([f"rfc{num}" for num in rfc_numbers]) + + +@shared_task +def load_rfcs_into_blobdb_task(start: int, end: int): + """Move file content for rfcs from rfc{start} to rfc{end} inclusive + + As this is expected to be removed once the blobdb is populated, it + will truncate its work to a coded max end. + This will not overwrite any existing blob content, and will only + log a small complaint if asked to load a non-exsiting RFC. + """ + # Protect us from ourselves + if end < start: + return + if start < 1: + start = 1 + if end > 11000: # Arbitrarily chosen + end = 11000 + load_rfcs_into_blobdb(list(range(start, end + 1))) + + +@shared_task +def create_rfc_index_task(): + create_rfc_txt_index() + create_rfc_xml_index() + diff --git a/ietf/sync/tests.py b/ietf/sync/tests.py index 182b6e24c4..21d6cb5cd5 100644 --- a/ietf/sync/tests.py +++ b/ietf/sync/tests.py @@ -206,7 +206,7 @@ def test_iana_review_mail(self): doc_name, review_time, by, comment = iana.parse_review_email(msg.encode('utf-8')) self.assertEqual(doc_name, draft.name) - self.assertEqual(review_time, datetime.datetime(2012, 5, 10, 12, 0, rtime, tzinfo=datetime.timezone.utc)) + self.assertEqual(review_time, datetime.datetime(2012, 5, 10, 12, 0, rtime, tzinfo=datetime.UTC)) self.assertEqual(by, Person.objects.get(user__username="iana")) self.assertIn("there are no IANA Actions", comment.replace("\n", "")) @@ -240,7 +240,7 @@ def test_ingest_review_email(self, mock_parse_review_email, mock_add_review_comm args = ( "doc-name", - datetime.datetime.now(tz=datetime.timezone.utc), + datetime.datetime.now(tz=datetime.UTC), PersonFactory(), "yadda yadda yadda", ) @@ -301,6 +301,7 @@ def test_rfc_index(self): ad=Person.objects.get(user__username='ad'), external_url="http://my-external-url.example.com", note="this is a note", + pages=54, # make sure this is not 42 ) DocumentAuthorFactory.create_batch(2, document=draft_doc) draft_doc.action_holders.add(draft_doc.ad) # not normally set, but add to be sure it's cleared @@ -446,7 +447,7 @@ def test_rfc_index(self): rfc_doc = Document.objects.filter(rfc_number=1234, type_id="rfc").first() self.assertIsNotNone(rfc_doc, "RFC document should have been created") - self.assertEqual(rfc_doc.authors(), draft_doc.authors()) + self.assertEqual(rfc_doc.author_persons_or_names(), draft_doc.author_persons_or_names()) rfc_events = rfc_doc.docevent_set.all() self.assertEqual(len(rfc_events), 8) expected_events = [ @@ -889,8 +890,9 @@ class TaskTests(TestCase): @mock.patch("ietf.sync.tasks.rfceditor.update_docs_from_rfc_index") @mock.patch("ietf.sync.tasks.rfceditor.parse_index") @mock.patch("ietf.sync.tasks.requests.get") + @mock.patch("ietf.sync.tasks.rsync_rfcs_from_rfceditor_task.delay") def test_rfc_editor_index_update_task( - self, requests_get_mock, parse_index_mock, update_docs_mock + self, rsync_task_mock, requests_get_mock, parse_index_mock, update_docs_mock ) -> None: # the annotation here prevents mypy from complaining about annotation-unchecked """rfc_editor_index_update_task calls helpers correctly @@ -922,6 +924,7 @@ def json(self): rfc = RfcFactory() # Test with full_index = False + rsync_task_mock.return_value = None requests_get_mock.side_effect = (index_response, errata_response) # will step through these parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS) update_docs_mock.return_value = ( @@ -947,10 +950,13 @@ def json(self): ) self.assertIsNotNone(update_docs_kwargs["skip_older_than_date"]) + self.assertFalse(rsync_task_mock.called) + # Test again with full_index = True requests_get_mock.reset_mock() parse_index_mock.reset_mock() update_docs_mock.reset_mock() + rsync_task_mock.reset_mock() requests_get_mock.side_effect = (index_response, errata_response) # will step through these tasks.rfc_editor_index_update_task(full_index=True) @@ -971,40 +977,67 @@ def json(self): ) self.assertIsNone(update_docs_kwargs["skip_older_than_date"]) + self.assertFalse(rsync_task_mock.called) + + # Test again where the index would cause a new RFC to come into existance + requests_get_mock.reset_mock() + parse_index_mock.reset_mock() + update_docs_mock.reset_mock() + rsync_task_mock.reset_mock() + requests_get_mock.side_effect = ( + index_response, + errata_response, + ) # will step through these + update_docs_mock.return_value = ( + (rfc.rfc_number, ("something changed",), rfc, True), + ) + tasks.rfc_editor_index_update_task(full_index=True) + self.assertTrue(rsync_task_mock.called) + rsync_task_args, rsync_task_kwargs = rsync_task_mock.call_args + self.assertEqual((([rfc.rfc_number],), {}), (rsync_task_args, rsync_task_kwargs)) + # Test error handling requests_get_mock.reset_mock() parse_index_mock.reset_mock() update_docs_mock.reset_mock() + rsync_task_mock.reset_mock() requests_get_mock.side_effect = requests.Timeout # timeout on every get() tasks.rfc_editor_index_update_task(full_index=False) self.assertFalse(parse_index_mock.called) self.assertFalse(update_docs_mock.called) + self.assertFalse(rsync_task_mock.called) requests_get_mock.reset_mock() parse_index_mock.reset_mock() update_docs_mock.reset_mock() + rsync_task_mock.reset_mock() requests_get_mock.side_effect = [index_response, requests.Timeout] # timeout second get() tasks.rfc_editor_index_update_task(full_index=False) self.assertFalse(update_docs_mock.called) + self.assertFalse(rsync_task_mock.called) requests_get_mock.reset_mock() parse_index_mock.reset_mock() update_docs_mock.reset_mock() + rsync_task_mock.reset_mock() requests_get_mock.side_effect = [index_response, errata_response] # feed in an index that is too short parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS - 1) tasks.rfc_editor_index_update_task(full_index=False) self.assertTrue(parse_index_mock.called) self.assertFalse(update_docs_mock.called) + self.assertFalse(rsync_task_mock.called) requests_get_mock.reset_mock() parse_index_mock.reset_mock() update_docs_mock.reset_mock() + rsync_task_mock.reset_mock() requests_get_mock.side_effect = [index_response, errata_response] errata_response.json_length = rfceditor.MIN_ERRATA_RESULTS - 1 # too short parse_index_mock.return_value = MockIndexData(length=rfceditor.MIN_INDEX_RESULTS) tasks.rfc_editor_index_update_task(full_index=False) self.assertFalse(update_docs_mock.called) + self.assertFalse(rsync_task_mock.called) @override_settings(RFC_EDITOR_QUEUE_URL="https://rfc-editor.example.com/queue/") @mock.patch("ietf.sync.tasks.update_drafts_from_queue") @@ -1121,7 +1154,7 @@ def test_iana_protocols_update_task( ) self.assertEqual( published_later_than, - {datetime.datetime(2012,11,26,tzinfo=datetime.timezone.utc)} + {datetime.datetime(2012,11,26,tzinfo=datetime.UTC)} ) # try with an exception @@ -1134,3 +1167,51 @@ def test_iana_protocols_update_task( self.assertTrue(requests_get_mock.called) self.assertFalse(parse_protocols_mock.called) self.assertFalse(update_rfc_log_mock.called) + + @mock.patch("ietf.sync.tasks.rsync_helper") + @mock.patch("ietf.sync.tasks.load_rfcs_into_blobdb") + @mock.patch("ietf.sync.tasks.rebuild_reference_relations_task.delay") + def test_rsync_rfcs_from_rfceditor_task( + self, + rebuild_relations_mock, + load_blobs_mock, + rsync_helper_mock, + ): + tasks.rsync_rfcs_from_rfceditor_task([12345, 54321]) + self.assertTrue(rsync_helper_mock.called) + self.assertTrue(load_blobs_mock.called) + load_blobs_args, load_blobs_kwargs = load_blobs_mock.call_args + self.assertEqual(load_blobs_args, ([12345, 54321],)) + self.assertEqual(load_blobs_kwargs, {}) + self.assertTrue(rebuild_relations_mock.called) + rebuild_args, rebuild_kwargs = rebuild_relations_mock.call_args + self.assertEqual(rebuild_args, (["rfc12345", "rfc54321"],)) + self.assertEqual(rebuild_kwargs, {}) + + @mock.patch("ietf.sync.tasks.load_rfcs_into_blobdb") + def test_load_rfcs_into_blobdb_task( + self, + load_blobs_mock, + ): + tasks.load_rfcs_into_blobdb_task(5, 3) + self.assertFalse(load_blobs_mock.called) + load_blobs_mock.reset_mock() + tasks.load_rfcs_into_blobdb_task(-1, 1) + self.assertTrue(load_blobs_mock.called) + mock_args, mock_kwargs = load_blobs_mock.call_args + self.assertEqual(mock_args, ([1],)) + self.assertEqual(mock_kwargs, {}) + load_blobs_mock.reset_mock() + tasks.load_rfcs_into_blobdb_task(10999, 50000) + self.assertTrue(load_blobs_mock.called) + mock_args, mock_kwargs = load_blobs_mock.call_args + self.assertEqual(mock_args, ([10999, 11000],)) + self.assertEqual(mock_kwargs, {}) + load_blobs_mock.reset_mock() + tasks.load_rfcs_into_blobdb_task(3261, 3263) + self.assertTrue(load_blobs_mock.called) + mock_args, mock_kwargs = load_blobs_mock.call_args + self.assertEqual(mock_args, ([3261, 3262, 3263],)) + self.assertEqual(mock_kwargs, {}) + + diff --git a/ietf/sync/tests_rfcindex.py b/ietf/sync/tests_rfcindex.py new file mode 100644 index 0000000000..e682c016f5 --- /dev/null +++ b/ietf/sync/tests_rfcindex.py @@ -0,0 +1,236 @@ +# Copyright The IETF Trust 2026, All Rights Reserved +import json +from unittest import mock + +from django.core.files.base import ContentFile +from django.core.files.storage import storages +from django.test.utils import override_settings +from lxml import etree + +from ietf.doc.factories import PublishedRfcDocEventFactory, IndividualRfcFactory +from ietf.name.models import DocTagName +from ietf.sync.rfcindex import ( + create_rfc_txt_index, + create_rfc_xml_index, + format_rfc_number, + save_to_red_bucket, + get_unusable_rfc_numbers, + get_april1_rfc_numbers, + get_publication_std_levels, +) +from ietf.utils.test_utils import TestCase + + +class RfcIndexTests(TestCase): + """Tests of rfc-index generation + + Tests are limited and should cover more cases. Needs: + * test of subseries docs + * test of related docs (obsoletes/updates + reverse directions) + * more thorough validation of index contents + + Be careful when calling create_rfc_txt_index() or create_rfc_xml_index(). These + will save to a storage by default, which can introduce cross-talk between tests. + Best to patch that method with a mock. + """ + + def setUp(self): + super().setUp() + red_bucket = storages["red_bucket"] + + # Create an unused RFC number + red_bucket.save( + "input/unusable-rfc-numbers.json", + ContentFile(json.dumps([{"number": 123, "comment": ""}])), + ) + + # actual April 1 RFC + self.april_fools_rfc = PublishedRfcDocEventFactory( + time="2020-04-01T12:00:00Z", + doc=IndividualRfcFactory( + name="rfc4560", + rfc_number=4560, + stream_id="ise", + std_level_id="inf", + ), + ).doc + # Set up a JSON file to flag the April 1 RFC + red_bucket.save( + "input/april-first-rfc-numbers.json", + ContentFile(json.dumps([self.april_fools_rfc.rfc_number])), + ) + + # non-April Fools RFC that happens to have been published on April 1 + self.rfc = PublishedRfcDocEventFactory( + time="2021-04-01T12:00:00Z", + doc__name="rfc10000", + doc__rfc_number=10000, + doc__std_level_id="std", + ).doc + self.rfc.tags.add(DocTagName.objects.get(slug="errata")) + + # Set up a publication-std-levels.json file to indicate the publication + # standard of self.rfc as different from its current value + red_bucket.save( + "input/publication-std-levels.json", + ContentFile( + json.dumps( + [{"number": self.rfc.rfc_number, "publication_std_level": "ps"}] + ) + ), + ) + + def tearDown(self): + red_bucket = storages["red_bucket"] + red_bucket.delete("input/unusable-rfc-numbers.json") + red_bucket.delete("input/april-first-rfc-numbers.json") + red_bucket.delete("input/publication-std-levels.json") + super().tearDown() + + @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") + def test_create_rfc_txt_index(self, mock_save): + create_rfc_txt_index() + self.assertEqual(mock_save.call_count, 1) + self.assertEqual(mock_save.call_args[0][0], "rfc-index.txt") + contents = mock_save.call_args[0][1] + self.assertTrue(isinstance(contents, str)) + self.assertIn( + "123 Not Issued.", + contents, + ) + # No zero prefix! + self.assertNotIn( + "0123 Not Issued.", + contents, + ) + self.assertIn( + f"{self.april_fools_rfc.rfc_number} {self.april_fools_rfc.title}", + contents, + ) + self.assertIn("1 April 2020", contents) # from the April 1 RFC + self.assertIn( + f"{self.rfc.rfc_number} {self.rfc.title}", + contents, + ) + self.assertIn("April 2021", contents) # from the non-April 1 RFC + self.assertNotIn("1 April 2021", contents) + + @override_settings(RFCINDEX_INPUT_PATH="input/") + @mock.patch("ietf.sync.rfcindex.save_to_red_bucket") + def test_create_rfc_xml_index(self, mock_save): + create_rfc_xml_index() + self.assertEqual(mock_save.call_count, 1) + self.assertEqual(mock_save.call_args[0][0], "rfc-index.xml") + contents = mock_save.call_args[0][1] + self.assertTrue(isinstance(contents, bytes)) + ns = "{https://www.rfc-editor.org/rfc-index}" # NOT an f-string + index = etree.fromstring(contents) + + # We can aspire to validating the schema - currently does not conform because + # XSD expects 4-digit RFC numbers (etc). + # + # xmlschema = etree.XMLSchema(etree.fromstring( + # Path(__file__).with_name("rfc-index.xsd").read_bytes()) + # ) + # xmlschema.assertValid(index) + + children = list(index) # elements as list + # Should be one rfc-not-issued-entry + self.assertEqual(len(children), 3) + self.assertEqual( + [ + c.find(f"{ns}doc-id").text + for c in children + if c.tag == f"{ns}rfc-not-issued-entry" + ], + ["RFC123"], + ) + # Should be two rfc-entries + rfc_entries = { + c.find(f"{ns}doc-id").text: c for c in children if c.tag == f"{ns}rfc-entry" + } + + # Check the April Fool's entry + april_fools_entry = rfc_entries[self.april_fools_rfc.name.upper()] + self.assertEqual( + april_fools_entry.find(f"{ns}title").text, + self.april_fools_rfc.title, + ) + self.assertEqual( + [(c.tag, c.text) for c in april_fools_entry.find(f"{ns}date")], + [(f"{ns}month", "April"), (f"{ns}day", "1"), (f"{ns}year", "2020")], + ) + self.assertEqual( + april_fools_entry.find(f"{ns}current-status").text, + "INFORMATIONAL", + ) + self.assertEqual( + april_fools_entry.find(f"{ns}publication-status").text, + "UNKNOWN", + ) + + # Check the Regular entry + rfc_entry = rfc_entries[self.rfc.name.upper()] + self.assertEqual(rfc_entry.find(f"{ns}title").text, self.rfc.title) + self.assertEqual( + rfc_entry.find(f"{ns}current-status").text, "INTERNET STANDARD" + ) + self.assertEqual( + rfc_entry.find(f"{ns}publication-status").text, "PROPOSED STANDARD" + ) + self.assertEqual( + [(c.tag, c.text) for c in rfc_entry.find(f"{ns}date")], + [(f"{ns}month", "April"), (f"{ns}year", "2021")], + ) + + +class HelperTests(TestCase): + def test_format_rfc_number(self): + self.assertEqual(format_rfc_number(10), "10") + with override_settings(RFCINDEX_MATCH_LEGACY_XML=True): + self.assertEqual(format_rfc_number(10), "0010") + + def test_save_to_red_bucket(self): + red_bucket = storages["red_bucket"] + with override_settings(RFCINDEX_DELETE_THEN_WRITE=False): + save_to_red_bucket("test", "contents \U0001f600") + # Read as binary and explicitly decode to confirm encoding + with red_bucket.open("test", "rb") as f: + self.assertEqual(f.read().decode("utf-8"), "contents \U0001f600") + with override_settings(RFCINDEX_DELETE_THEN_WRITE=True): + save_to_red_bucket("test", "new contents \U0001fae0".encode("utf-8")) + # Read as binary and explicitly decode to confirm encoding + with red_bucket.open("test", "rb") as f: + self.assertEqual(f.read().decode("utf-8"), "new contents \U0001fae0") + red_bucket.delete("test") # clean up like a good child + + def test_get_unusable_rfc_numbers_raises(self): + """get_unusable_rfc_numbers should bail on errors""" + with self.assertRaises(FileNotFoundError): + get_unusable_rfc_numbers() + red_bucket = storages["red_bucket"] + red_bucket.save("unusable-rfc-numbers.json", ContentFile("not json")) + with self.assertRaises(json.JSONDecodeError): + get_unusable_rfc_numbers() + red_bucket.delete("unusable-rfc-numbers.json") + + def test_get_april1_rfc_numbers_raises(self): + """get_april1_rfc_numbers should bail on errors""" + with self.assertRaises(FileNotFoundError): + get_april1_rfc_numbers() + red_bucket = storages["red_bucket"] + red_bucket.save("april-first-rfc-numbers.json", ContentFile("not json")) + with self.assertRaises(json.JSONDecodeError): + get_april1_rfc_numbers() + red_bucket.delete("april-first-rfc-numbers.json") + + def test_get_publication_std_levels_raises(self): + """get_publication_std_levels should bail on errors""" + with self.assertRaises(FileNotFoundError): + get_publication_std_levels() + red_bucket = storages["red_bucket"] + red_bucket.save("publication-std-levels.json", ContentFile("not json")) + with self.assertRaises(json.JSONDecodeError): + get_publication_std_levels() + red_bucket.delete("publication-std-levels.json") diff --git a/ietf/sync/tests_utils.py b/ietf/sync/tests_utils.py new file mode 100644 index 0000000000..bb4a859e30 --- /dev/null +++ b/ietf/sync/tests_utils.py @@ -0,0 +1,84 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +from pathlib import Path +from tempfile import TemporaryDirectory + +from django.test import override_settings +from ietf import settings +from ietf.doc.factories import RfcFactory +from ietf.doc.storage_utils import exists_in_storage, retrieve_str +from ietf.sync.utils import build_from_file_content, load_rfcs_into_blobdb, rsync_helper +from ietf.utils.test_utils import TestCase + + +class RsyncHelperTests(TestCase): + def test_rsync_helper(self): + with ( + TemporaryDirectory() as source_dir, + TemporaryDirectory() as dest_dir, + ): + with (Path(source_dir) / "canary.txt").open("w") as canary_source_file: + canary_source_file.write("chirp") + rsync_helper( + [ + "-a", + f"{source_dir}/", + f"{dest_dir}/", + ] + ) + with (Path(dest_dir) / "canary.txt").open("r") as canary_dest_file: + chirp = canary_dest_file.read() + self.assertEqual(chirp, "chirp") + + def test_build_from_file_content(self): + content = build_from_file_content([12345, 54321]) + self.assertEqual( + content, + """prerelease/ +rfc12345.txt +rfc12345.html +rfc12345.xml +rfc12345.pdf +rfc12345.ps +rfc12345.json +prerelease/rfc12345.notprepped.xml +rfc54321.txt +rfc54321.html +rfc54321.xml +rfc54321.pdf +rfc54321.ps +rfc54321.json +prerelease/rfc54321.notprepped.xml +""", + ) + + +class RfcBlobUploadTests(TestCase): + def test_load_rfcs_into_blobdb(self): + with TemporaryDirectory() as faux_rfc_path: + with override_settings(RFC_PATH=faux_rfc_path): + rfc_path = Path(faux_rfc_path) + (rfc_path / "prerelease").mkdir() + for num in [12345, 54321]: + RfcFactory(rfc_number=num) + for ext in settings.RFC_FILE_TYPES + ("json",): + with (rfc_path / f"rfc{num}.{ext}").open("w") as f: + f.write(ext) + with (rfc_path / "rfc{num}.bogon").open("w") as f: + f.write("bogon") + with (rfc_path / "prerelease" / f"rfc{num}.notprepped.xml").open( + "w" + ) as f: + f.write("notprepped") + load_rfcs_into_blobdb([12345, 54321]) + for num in [12345, 54321]: + for ext in settings.RFC_FILE_TYPES + ("json",): + self.assertEqual( + retrieve_str("rfc", f"{ext}/rfc{num}.{ext}"), + ext, + ) + self.assertFalse(exists_in_storage("rfc", f"bogon/rfc{num}.bogon")) + self.assertEqual( + retrieve_str("rfc", f"notprepped/rfc{num}.notprepped.xml"), + "notprepped", + ) diff --git a/ietf/sync/utils.py b/ietf/sync/utils.py new file mode 100644 index 0000000000..b3bdd8d206 --- /dev/null +++ b/ietf/sync/utils.py @@ -0,0 +1,77 @@ +# Copyright The IETF Trust 2026, All Rights Reserved + +import datetime +import subprocess + +from pathlib import Path + +from django.conf import settings +from ietf.utils import log +from ietf.doc.models import Document +from ietf.doc.storage_utils import AlreadyExistsError, store_bytes + + +def rsync_helper(subprocess_arg_array: list[str]): + subprocess.run(["/usr/bin/rsync"]+subprocess_arg_array) + +def build_from_file_content(rfc_numbers: list[int]) -> str: + types_to_sync = settings.RFC_FILE_TYPES + ("json",) + lines = [] + lines.append("prerelease/") + for num in rfc_numbers: + for ext in types_to_sync: + lines.append(f"rfc{num}.{ext}") + lines.append(f"prerelease/rfc{num}.notprepped.xml") + return "\n".join(lines)+"\n" + +def load_rfcs_into_blobdb(numbers: list[int]): + types_to_load = settings.RFC_FILE_TYPES + ("json",) + rfc_docs = Document.objects.filter(type="rfc", rfc_number__in=numbers).values_list("rfc_number", flat=True) + for num in numbers: + if num in rfc_docs: + for ext in types_to_load: + fs_path = Path(settings.RFC_PATH) / f"rfc{num}.{ext}" + if fs_path.is_file(): + with fs_path.open("rb") as f: + bytes = f.read() + mtime = fs_path.stat().st_mtime + try: + store_bytes( + kind="rfc", + name=f"{ext}/rfc{num}.{ext}", + content=bytes, + allow_overwrite=False, # Intentionally not allowing overwrite. + doc_name=f"rfc{num}", + doc_rev=None, + # Not setting content_type + mtime=datetime.datetime.fromtimestamp( + mtime, tz=datetime.UTC + ), + ) + except AlreadyExistsError as e: + log.log(str(e)) + + # store the not-prepped xml + name = f"rfc{num}.notprepped.xml" + source = Path(settings.RFC_PATH) / "prerelease" / name + if source.is_file(): + with open(source, "rb") as f: + bytes = f.read() + mtime = source.stat().st_mtime + try: + store_bytes( + kind="rfc", + name=f"notprepped/{name}", + content=bytes, + allow_overwrite=False, # Intentionally not allowing overwrite. + doc_name=f"rfc{num}", + doc_rev=None, + # Not setting content_type + mtime=datetime.datetime.fromtimestamp(mtime, tz=datetime.UTC), + ) + except AlreadyExistsError as e: + log.log(str(e)) + else: + log.log( + f"Skipping loading rfc{num} into blobdb as no matching Document exists" + ) diff --git a/ietf/templates/base.html b/ietf/templates/base.html index aa44955527..25ce50c467 100644 --- a/ietf/templates/base.html +++ b/ietf/templates/base.html @@ -15,6 +15,7 @@ {% block title %}No title{% endblock %} + @@ -96,7 +97,7 @@
{% endif %}
-